Replace superuser role for API keys (#81977)

This PR replace superuser role of an API key with the new limited
superuser role to prevent write access to system indices.

The replacement should only happen to the builtin superuser role.
If there is a user-created role that has the exact same definition as
the superuser role, it will not be replaced because we consider users
are explicitly opt-in for ALL access in this scenario.

Relates: #81400

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Subject.java

@@ -133,12 +133,15 @@ private List<RoleReference> buildRoleReferencesForApiKey() {
         final RoleReference.ApiKeyRoleReference limitedByRoleReference = new RoleReference.ApiKeyRoleReference(
             apiKeyId,
             limitedByRoleDescriptorsBytes,
-            "apikey_limited_role"
+            RoleReference.ApiKeyRoleType.LIMITED_BY
         );
         if (isEmptyRoleDescriptorsBytes(roleDescriptorsBytes)) {
             return List.of(limitedByRoleReference);
         }
-        return List.of(new RoleReference.ApiKeyRoleReference(apiKeyId, roleDescriptorsBytes, "apikey_role"), limitedByRoleReference);
+        return List.of(
+            new RoleReference.ApiKeyRoleReference(apiKeyId, roleDescriptorsBytes, RoleReference.ApiKeyRoleType.ASSIGNED),
+            limitedByRoleReference
+        );
     }
 
     private boolean isEmptyRoleDescriptorsBytes(BytesReference roleDescriptorsBytes) {
@@ -155,13 +158,13 @@ private List<RoleReference> buildRolesReferenceForApiKeyBwc() {
             final RoleReference.BwcApiKeyRoleReference limitedByRoleReference = new RoleReference.BwcApiKeyRoleReference(
                 apiKeyId,
                 limitedByRoleDescriptorsMap,
-                "_limited_role_desc"
+                RoleReference.ApiKeyRoleType.LIMITED_BY
             );
             if (roleDescriptorsMap == null || roleDescriptorsMap.isEmpty()) {
                 return List.of(limitedByRoleReference);
             } else {
                 return List.of(
-                    new RoleReference.BwcApiKeyRoleReference(apiKeyId, roleDescriptorsMap, "_role_desc"),
+                    new RoleReference.BwcApiKeyRoleReference(apiKeyId, roleDescriptorsMap, RoleReference.ApiKeyRoleType.ASSIGNED),
                     limitedByRoleReference
                 );
             }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/RoleReference.java

@@ -77,13 +77,13 @@ final class ApiKeyRoleReference implements RoleReference {
 
         private final String apiKeyId;
         private final BytesReference roleDescriptorsBytes;
-        private final String roleKeySource;
+        private final ApiKeyRoleType roleType;
         private RoleKey id = null;
 
-        public ApiKeyRoleReference(String apiKeyId, BytesReference roleDescriptorsBytes, String roleKeySource) {
+        public ApiKeyRoleReference(String apiKeyId, BytesReference roleDescriptorsBytes, ApiKeyRoleType roleType) {
             this.apiKeyId = apiKeyId;
             this.roleDescriptorsBytes = roleDescriptorsBytes;
-            this.roleKeySource = roleKeySource;
+            this.roleType = roleType;
         }
 
         @Override
@@ -93,7 +93,7 @@ public RoleKey id() {
                 final String roleDescriptorsHash = MessageDigests.toHexString(
                     MessageDigests.digest(roleDescriptorsBytes, MessageDigests.sha256())
                 );
-                id = new RoleKey(Set.of("apikey:" + roleDescriptorsHash), roleKeySource);
+                id = new RoleKey(Set.of("apikey:" + roleDescriptorsHash), "apikey_" + roleType);
             }
             return id;
         }
@@ -110,6 +110,10 @@ public String getApiKeyId() {
         public BytesReference getRoleDescriptorsBytes() {
             return roleDescriptorsBytes;
         }
+
+        public ApiKeyRoleType getRoleType() {
+            return roleType;
+        }
     }
 
     /**
@@ -118,18 +122,18 @@ public BytesReference getRoleDescriptorsBytes() {
     final class BwcApiKeyRoleReference implements RoleReference {
         private final String apiKeyId;
         private final Map<String, Object> roleDescriptorsMap;
-        private final String roleKeySourceSuffix;
+        private final ApiKeyRoleType roleType;
 
-        public BwcApiKeyRoleReference(String apiKeyId, Map<String, Object> roleDescriptorsMap, String roleKeySourceSuffix) {
+        public BwcApiKeyRoleReference(String apiKeyId, Map<String, Object> roleDescriptorsMap, ApiKeyRoleType roleType) {
             this.apiKeyId = apiKeyId;
             this.roleDescriptorsMap = roleDescriptorsMap;
-            this.roleKeySourceSuffix = roleKeySourceSuffix;
+            this.roleType = roleType;
         }
 
         @Override
         public RoleKey id() {
             // Since api key id is unique, it is sufficient and more correct to use it as the names
-            return new RoleKey(Set.of(apiKeyId), "bwc_api_key" + roleKeySourceSuffix);
+            return new RoleKey(Set.of(apiKeyId), "bwc_api_key_" + roleType);
         }
 
         @Override
@@ -144,6 +148,10 @@ public String getApiKeyId() {
         public Map<String, Object> getRoleDescriptorsMap() {
             return roleDescriptorsMap;
         }
+
+        public ApiKeyRoleType getRoleType() {
+            return roleType;
+        }
     }
 
     /**
@@ -170,4 +178,18 @@ public void resolve(RoleReferenceResolver resolver, ActionListener<RolesRetrieva
             resolver.resolveServiceAccountRoleReference(this, listener);
         }
     }
+
+    /**
+     * The type of one set of API key roles.
+     */
+    enum ApiKeyRoleType {
+        /**
+         * Roles directly specified by the creator user on API key creation
+         */
+        ASSIGNED,
+        /**
+         * Roles captured for the owner user as the upper bound of the assigned roles
+         */
+        LIMITED_BY;
+    }
 }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/RoleReferenceTests.java

@@ -50,19 +50,19 @@ public void testSuperuserRoleReference() {
     public void testApiKeyRoleReference() {
         final String apiKeyId = randomAlphaOfLength(20);
         final BytesArray roleDescriptorsBytes = new BytesArray(randomAlphaOfLength(50));
-        final String roleKeySource = randomAlphaOfLength(8);
+        final RoleReference.ApiKeyRoleType apiKeyRoleType = randomFrom(RoleReference.ApiKeyRoleType.values());
         final RoleReference.ApiKeyRoleReference apiKeyRoleReference = new RoleReference.ApiKeyRoleReference(
             apiKeyId,
             roleDescriptorsBytes,
-            roleKeySource
+            apiKeyRoleType
         );
 
         final RoleKey roleKey = apiKeyRoleReference.id();
         assertThat(
             roleKey.getNames(),
             hasItem("apikey:" + MessageDigests.toHexString(MessageDigests.digest(roleDescriptorsBytes, MessageDigests.sha256())))
         );
-        assertThat(roleKey.getSource(), equalTo(roleKeySource));
+        assertThat(roleKey.getSource(), equalTo("apikey_" + apiKeyRoleType));
     }
 
     public void testServiceAccountRoleReference() {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java

@@ -92,6 +92,9 @@
 import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
+import org.elasticsearch.xpack.core.security.authz.store.RoleReference;
+import org.elasticsearch.xpack.core.security.support.MetadataUtils;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.support.CacheInvalidatorRegistry;
 import org.elasticsearch.xpack.security.support.FeatureNotEnabledException;
@@ -519,11 +522,15 @@ void loadApiKeyAndValidateCredentials(
         }), client::get);
     }
 
-    public List<RoleDescriptor> parseRoleDescriptors(final String apiKeyId, final Map<String, Object> roleDescriptors) {
-        if (roleDescriptors == null) {
+    public List<RoleDescriptor> parseRoleDescriptors(
+        final String apiKeyId,
+        final Map<String, Object> roleDescriptorsMap,
+        RoleReference.ApiKeyRoleType roleType
+    ) {
+        if (roleDescriptorsMap == null) {
             return null;
         }
-        return roleDescriptors.entrySet().stream().map(entry -> {
+        final List<RoleDescriptor> roleDescriptors = roleDescriptorsMap.entrySet().stream().map(entry -> {
             final String name = entry.getKey();
             @SuppressWarnings("unchecked")
             final Map<String, Object> rdMap = (Map<String, Object>) entry.getValue();
@@ -543,9 +550,16 @@ public List<RoleDescriptor> parseRoleDescriptors(final String apiKeyId, final Ma
                 throw new UncheckedIOException(e);
             }
         }).collect(Collectors.toList());
+        return roleType == RoleReference.ApiKeyRoleType.LIMITED_BY
+            ? maybeReplaceSuperuserRoleDescriptor(apiKeyId, roleDescriptors)
+            : roleDescriptors;
     }
 
-    public List<RoleDescriptor> parseRoleDescriptorsBytes(final String apiKeyId, BytesReference bytesReference) {
+    public List<RoleDescriptor> parseRoleDescriptorsBytes(
+        final String apiKeyId,
+        BytesReference bytesReference,
+        RoleReference.ApiKeyRoleType roleType
+    ) {
         if (bytesReference == null) {
             return Collections.emptyList();
         }
@@ -568,7 +582,42 @@ public List<RoleDescriptor> parseRoleDescriptorsBytes(final String apiKeyId, Byt
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
-        return Collections.unmodifiableList(roleDescriptors);
+        return roleType == RoleReference.ApiKeyRoleType.LIMITED_BY
+            ? maybeReplaceSuperuserRoleDescriptor(apiKeyId, roleDescriptors)
+            : roleDescriptors;
+    }
+
+    // package private for tests
+    static final RoleDescriptor LEGACY_SUPERUSER_ROLE_DESCRIPTOR = new RoleDescriptor(
+        "superuser",
+        new String[] { "all" },
+        new RoleDescriptor.IndicesPrivileges[] {
+            RoleDescriptor.IndicesPrivileges.builder().indices("*").privileges("all").allowRestrictedIndices(true).build() },
+        new RoleDescriptor.ApplicationResourcePrivileges[] {
+            RoleDescriptor.ApplicationResourcePrivileges.builder().application("*").privileges("*").resources("*").build() },
+        null,
+        new String[] { "*" },
+        MetadataUtils.DEFAULT_RESERVED_METADATA,
+        Collections.emptyMap()
+    );
+
+    // This method should only be called to replace the superuser role descriptor for the limited-by roles of an API Key.
+    // We do not replace assigned roles because they are created explicitly by users.
+    // Before #82049, it is possible to specify a role descriptor for API keys that is identical to the builtin superuser role
+    // (including the _reserved metadata field).
+    private List<RoleDescriptor> maybeReplaceSuperuserRoleDescriptor(String apiKeyId, List<RoleDescriptor> roleDescriptors) {
+        // Scan through all the roles because superuser can be one of the roles that a user has. Unlike building the Role object,
+        // capturing role descriptors does not preempt for superuser.
+        return roleDescriptors.stream().map(rd -> {
+            // Since we are only replacing limited-by roles and all limited-by roles are looked up with role providers,
+            // it is technically possible to just check the name of superuser and the _reserved metadata field.
+            // But the gain is not much since role resolving is cached and comparing the whole role descriptor is still safer.
+            if (rd.equals(LEGACY_SUPERUSER_ROLE_DESCRIPTOR)) {
+                logger.debug("replacing superuser role for API key [{}]", apiKeyId);
+                return ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR;
+            }
+            return rd;
+        }).toList();
     }
 
     /**

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/RoleDescriptorStore.java

@@ -99,7 +99,8 @@ public void resolveApiKeyRoleReference(
     ) {
         final List<RoleDescriptor> roleDescriptors = apiKeyService.parseRoleDescriptorsBytes(
             apiKeyRoleReference.getApiKeyId(),
-            apiKeyRoleReference.getRoleDescriptorsBytes()
+            apiKeyRoleReference.getRoleDescriptorsBytes(),
+            apiKeyRoleReference.getRoleType()
         );
         final RolesRetrievalResult rolesRetrievalResult = new RolesRetrievalResult();
         rolesRetrievalResult.addDescriptors(Set.copyOf(roleDescriptors));
@@ -113,7 +114,8 @@ public void resolveBwcApiKeyRoleReference(
     ) {
         final List<RoleDescriptor> roleDescriptors = apiKeyService.parseRoleDescriptors(
             bwcApiKeyRoleReference.getApiKeyId(),
-            bwcApiKeyRoleReference.getRoleDescriptorsMap()
+            bwcApiKeyRoleReference.getRoleDescriptorsMap(),
+            bwcApiKeyRoleReference.getRoleType()
         );
         final RolesRetrievalResult rolesRetrievalResult = new RolesRetrievalResult();
         rolesRetrievalResult.addDescriptors(Set.copyOf(roleDescriptors));

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java

@@ -66,6 +66,7 @@
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
+import org.elasticsearch.xpack.core.security.authz.store.RoleReference;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.authc.ApiKeyService.ApiKeyCredentials;
 import org.elasticsearch.xpack.security.authc.ApiKeyService.ApiKeyDoc;
@@ -111,6 +112,7 @@
 import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.INTERNAL_SECURITY_MAIN_INDEX_7;
 import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS;
 import static org.elasticsearch.xpack.security.Security.SECURITY_CRYPTO_THREAD_POOL_NAME;
+import static org.elasticsearch.xpack.security.authc.ApiKeyService.LEGACY_SUPERUSER_ROLE_DESCRIPTOR;
 import static org.hamcrest.Matchers.anEmptyMap;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
@@ -578,8 +580,8 @@ public void testParseRoleDescriptorsMap() throws Exception {
         }).when(privilegesStore).getPrivileges(any(Collection.class), any(Collection.class), anyActionListener());
         ApiKeyService service = createApiKeyService(Settings.EMPTY);
 
-        assertThat(service.parseRoleDescriptors(apiKeyId, null), nullValue());
-        assertThat(service.parseRoleDescriptors(apiKeyId, Collections.emptyMap()), emptyIterable());
+        assertThat(service.parseRoleDescriptors(apiKeyId, null, randomApiKeyRoleType()), nullValue());
+        assertThat(service.parseRoleDescriptors(apiKeyId, Collections.emptyMap(), randomApiKeyRoleType()), emptyIterable());
 
         final RoleDescriptor roleARoleDescriptor = new RoleDescriptor(
             "a role",
@@ -597,7 +599,7 @@ public void testParseRoleDescriptorsMap() throws Exception {
             );
         }
 
-        List<RoleDescriptor> roleDescriptors = service.parseRoleDescriptors(apiKeyId, Map.of("a role", roleARDMap));
+        List<RoleDescriptor> roleDescriptors = service.parseRoleDescriptors(apiKeyId, Map.of("a role", roleARDMap), randomApiKeyRoleType());
         assertThat(roleDescriptors, hasSize(1));
         assertThat(roleDescriptors.get(0), equalTo(roleARoleDescriptor));
 
@@ -609,19 +611,45 @@ public void testParseRoleDescriptorsMap() throws Exception {
                 false
             );
         }
-        roleDescriptors = service.parseRoleDescriptors(apiKeyId, Map.of(SUPERUSER_ROLE_DESCRIPTOR.getName(), superUserRdMap));
+        roleDescriptors = service.parseRoleDescriptors(
+            apiKeyId,
+            Map.of(SUPERUSER_ROLE_DESCRIPTOR.getName(), superUserRdMap),
+            randomApiKeyRoleType()
+        );
         assertThat(roleDescriptors, hasSize(1));
         assertThat(roleDescriptors.get(0), equalTo(SUPERUSER_ROLE_DESCRIPTOR));
+
+        final Map<String, Object> legacySuperUserRdMap;
+        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+            legacySuperUserRdMap = XContentHelper.convertToMap(
+                XContentType.JSON.xContent(),
+                BytesReference.bytes(LEGACY_SUPERUSER_ROLE_DESCRIPTOR.toXContent(builder, ToXContent.EMPTY_PARAMS, true)).streamInput(),
+                false
+            );
+        }
+        final RoleReference.ApiKeyRoleType apiKeyRoleType = randomApiKeyRoleType();
+        roleDescriptors = service.parseRoleDescriptors(
+            apiKeyId,
+            Map.of(LEGACY_SUPERUSER_ROLE_DESCRIPTOR.getName(), legacySuperUserRdMap),
+            apiKeyRoleType
+        );
+        assertThat(roleDescriptors, hasSize(1));
+        assertThat(
+            roleDescriptors.get(0),
+            equalTo(
+                apiKeyRoleType == RoleReference.ApiKeyRoleType.LIMITED_BY ? SUPERUSER_ROLE_DESCRIPTOR : LEGACY_SUPERUSER_ROLE_DESCRIPTOR
+            )
+        );
     }
 
     public void testParseRoleDescriptors() {
         ApiKeyService service = createApiKeyService(Settings.EMPTY);
         final String apiKeyId = randomAlphaOfLength(12);
-        List<RoleDescriptor> roleDescriptors = service.parseRoleDescriptorsBytes(apiKeyId, null);
+        List<RoleDescriptor> roleDescriptors = service.parseRoleDescriptorsBytes(apiKeyId, null, randomApiKeyRoleType());
         assertTrue(roleDescriptors.isEmpty());
 
         BytesReference roleBytes = new BytesArray("{\"a role\": {\"cluster\": [\"all\"]}}");
-        roleDescriptors = service.parseRoleDescriptorsBytes(apiKeyId, roleBytes);
+        roleDescriptors = service.parseRoleDescriptorsBytes(apiKeyId, roleBytes, randomApiKeyRoleType());
         assertEquals(1, roleDescriptors.size());
         assertEquals("a role", roleDescriptors.get(0).getName());
         assertArrayEquals(new String[] { "all" }, roleDescriptors.get(0).getClusterPrivileges());
@@ -635,12 +663,19 @@ public void testParseRoleDescriptors() {
                 + "\"privileges\":[\"*\"],\"resources\":[\"*\"]}],\"run_as\":[\"*\"],\"metadata\":{\"_reserved\":true},"
                 + "\"transient_metadata\":{}}}\n"
         );
-        roleDescriptors = service.parseRoleDescriptorsBytes(apiKeyId, roleBytes);
+        final RoleReference.ApiKeyRoleType apiKeyRoleType = randomApiKeyRoleType();
+        roleDescriptors = service.parseRoleDescriptorsBytes(apiKeyId, roleBytes, apiKeyRoleType);
         assertEquals(2, roleDescriptors.size());
         assertEquals(
             Set.of("reporting_user", "superuser"),
             roleDescriptors.stream().map(RoleDescriptor::getName).collect(Collectors.toSet())
         );
+        assertThat(
+            roleDescriptors.get(1),
+            equalTo(
+                apiKeyRoleType == RoleReference.ApiKeyRoleType.LIMITED_BY ? SUPERUSER_ROLE_DESCRIPTOR : LEGACY_SUPERUSER_ROLE_DESCRIPTOR
+            )
+        );
     }
 
     public void testApiKeyServiceDisabled() throws Exception {
@@ -1037,7 +1072,11 @@ public void testApiKeyDocCache() throws IOException, ExecutionException, Interru
         final BytesReference limitedByRoleDescriptorsBytes = service.getRoleDescriptorsBytesCache()
             .get(cachedApiKeyDoc.limitedByRoleDescriptorsHash);
         assertNotNull(limitedByRoleDescriptorsBytes);
-        final List<RoleDescriptor> limitedByRoleDescriptors = service.parseRoleDescriptorsBytes(docId, limitedByRoleDescriptorsBytes);
+        final List<RoleDescriptor> limitedByRoleDescriptors = service.parseRoleDescriptorsBytes(
+            docId,
+            limitedByRoleDescriptorsBytes,
+            RoleReference.ApiKeyRoleType.LIMITED_BY
+        );
         assertEquals(1, limitedByRoleDescriptors.size());
         assertEquals(SUPERUSER_ROLE_DESCRIPTOR, limitedByRoleDescriptors.get(0));
         if (metadata == null) {
@@ -1610,4 +1649,8 @@ private void checkAuthApiKeyMetadata(Object metadata, AuthenticationResult<User>
             );
         }
     }
+
+    private RoleReference.ApiKeyRoleType randomApiKeyRoleType() {
+        return randomFrom(RoleReference.ApiKeyRoleType.values());
+    }
 }