Allow custom authorization with an authorization engine (#38358)

For some users, the built in authorization mechanism does not fit their
needs and no feature that we offer would allow them to control the
authorization process to meet their needs. In order to support this,
a concept of an AuthorizationEngine is being introduced, which can be
provided using the security extension mechanism.

An AuthorizationEngine is responsible for making the authorization
decisions about a request. The engine is responsible for knowing how to
authorize and can be backed by whatever mechanism a user wants. The
default mechanism is one backed by roles to provide the authorization
decisions. The AuthorizationEngine will be called by the
AuthorizationService, which handles more of the internal workings that
apply in general to authorization within Elasticsearch.

In order to support external authorization services that would back an
authorization engine, the entire authorization process has become
asynchronous, which also includes all calls to the AuthorizationEngine.

The use of roles also leaked out of the AuthorizationService in our
existing code that is not specifically related to roles so this also
needed to be addressed. RequestInterceptor instances sometimes used a
role to ensure a user was not attempting to escalate their privileges.
Addressing this leakage of roles meant that the RequestInterceptor
execution needed to move within the AuthorizationService and that
AuthorizationEngines needed to support detection of whether a user has
more privileges on a name than another. The second area where roles
leaked to the user is in the handling of a few privilege APIs that
could be used to retrieve the user's privileges or ask if a user has
privileges to perform an action. To remove the leakage of roles from
these actions, the AuthorizationService and AuthorizationEngine gained
methods that enabled an AuthorizationEngine to return the response for
these APIs.

Ultimately this feature is the work included in:
#37785
#37495
#37328
#36245
#38137
#38219

Closes #32435

build.gradle

@@ -233,6 +233,9 @@ allprojects {
     "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}": ':modules:aggs-matrix-stats',
     "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
     "org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval',
+    // for security example plugins
+    "org.elasticsearch.plugin:x-pack-core:${version}": ':x-pack:plugin:core',
+    "org.elasticsearch.client.x-pack-transport:${version}": ':x-pack:transport-client'
   ]
 
   /*

plugins/examples/security-authorization-engine/build.gradle

@@ -0,0 +1,46 @@
+apply plugin: 'elasticsearch.esplugin'
+
+esplugin {
+  name 'security-authorization-engine'
+  description 'An example spi extension plugin for security that implements an Authorization Engine'
+  classname 'org.elasticsearch.example.AuthorizationEnginePlugin'
+  extendedPlugins = ['x-pack-security']
+}
+
+dependencies {
+  compileOnly "org.elasticsearch.plugin:x-pack-core:${version}"
+  testCompile "org.elasticsearch.client.x-pack-transport:${version}"
+}
+
+
+integTestRunner {
+    systemProperty 'tests.security.manager', 'false'
+}
+
+integTestCluster {
+  dependsOn buildZip
+  setting 'xpack.security.enabled', 'true'
+  setting 'xpack.ilm.enabled', 'false'
+  setting 'xpack.ml.enabled', 'false'
+  setting 'xpack.monitoring.enabled', 'false'
+  setting 'xpack.license.self_generated.type', 'trial'
+
+  // This is important, so that all the modules are available too.
+  // There are index templates that use token filters that are in analysis-module and
+  // processors are being used that are in ingest-common module.
+  distribution = 'default'
+
+  setupCommand 'setupDummyUser',
+               'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'custom_superuser'
+  waitCondition = { node, ant ->
+    File tmpFile = new File(node.cwd, 'wait.success')
+    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
+            dest: tmpFile.toString(),
+            username: 'test_user',
+            password: 'x-pack-test-password',
+            ignoreerrors: true,
+            retries: 10)
+    return tmpFile.exists()
+  }
+}
+check.dependsOn integTest

plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/AuthorizationEnginePlugin.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.example;
+
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.Plugin;
+
+/**
+ * Plugin class that is required so that the code contained here may be loaded as a plugin.
+ * Additional items such as settings and actions can be registered using this plugin class.
+ */
+public class AuthorizationEnginePlugin extends Plugin implements ActionPlugin {
+}

plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/CustomAuthorizationEngine.java

@@ -0,0 +1,238 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.example;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.metadata.AliasOrIndex;
+import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesResponse;
+import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesResponse.Indices;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest;
+import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
+import org.elasticsearch.xpack.core.security.authz.ResolvedIndices;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges;
+import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
+import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl.IndexAccessControl;
+import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions;
+import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivileges;
+import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
+import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege;
+import org.elasticsearch.xpack.core.security.user.User;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * A custom implementation of an authorization engine. This engine is extremely basic in that it
+ * authorizes based upon the name of a single role. If users have this role they are granted access.
+ */
+public class CustomAuthorizationEngine implements AuthorizationEngine {
+
+    @Override
+    public void resolveAuthorizationInfo(RequestInfo requestInfo, ActionListener<AuthorizationInfo> listener) {
+        final Authentication authentication = requestInfo.getAuthentication();
+        if (authentication.getUser().isRunAs()) {
+            final CustomAuthorizationInfo authenticatedUserAuthzInfo =
+                new CustomAuthorizationInfo(authentication.getUser().authenticatedUser().roles(), null);
+            listener.onResponse(new CustomAuthorizationInfo(authentication.getUser().roles(), authenticatedUserAuthzInfo));
+        } else {
+            listener.onResponse(new CustomAuthorizationInfo(authentication.getUser().roles(), null));
+        }
+    }
+
+    @Override
+    public void authorizeRunAs(RequestInfo requestInfo, AuthorizationInfo authorizationInfo, ActionListener<AuthorizationResult> listener) {
+        if (isSuperuser(requestInfo.getAuthentication().getUser().authenticatedUser())) {
+            listener.onResponse(AuthorizationResult.granted());
+        } else {
+            listener.onResponse(AuthorizationResult.deny());
+        }
+    }
+
+    @Override
+    public void authorizeClusterAction(RequestInfo requestInfo, AuthorizationInfo authorizationInfo,
+                                       ActionListener<AuthorizationResult> listener) {
+        if (isSuperuser(requestInfo.getAuthentication().getUser())) {
+            listener.onResponse(AuthorizationResult.granted());
+        } else {
+            listener.onResponse(AuthorizationResult.deny());
+        }
+    }
+
+    @Override
+    public void authorizeIndexAction(RequestInfo requestInfo, AuthorizationInfo authorizationInfo,
+                                     AsyncSupplier<ResolvedIndices> indicesAsyncSupplier,
+                                     Map<String, AliasOrIndex> aliasOrIndexLookup,
+                                     ActionListener<IndexAuthorizationResult> listener) {
+        if (isSuperuser(requestInfo.getAuthentication().getUser())) {
+            indicesAsyncSupplier.getAsync(ActionListener.wrap(resolvedIndices -> {
+                Map<String, IndexAccessControl> indexAccessControlMap = new HashMap<>();
+                for (String name : resolvedIndices.getLocal()) {
+                    indexAccessControlMap.put(name, new IndexAccessControl(true, FieldPermissions.DEFAULT, null));
+                }
+                IndicesAccessControl indicesAccessControl =
+                    new IndicesAccessControl(true, Collections.unmodifiableMap(indexAccessControlMap));
+                listener.onResponse(new IndexAuthorizationResult(true, indicesAccessControl));
+            }, listener::onFailure));
+        } else {
+            listener.onResponse(new IndexAuthorizationResult(true, IndicesAccessControl.DENIED));
+        }
+    }
+
+    @Override
+    public void loadAuthorizedIndices(RequestInfo requestInfo, AuthorizationInfo authorizationInfo,
+                                      Map<String, AliasOrIndex> aliasOrIndexLookup, ActionListener<List<String>> listener) {
+        if (isSuperuser(requestInfo.getAuthentication().getUser())) {
+            listener.onResponse(new ArrayList<>(aliasOrIndexLookup.keySet()));
+        } else {
+            listener.onResponse(Collections.emptyList());
+        }
+    }
+
+    @Override
+    public void validateIndexPermissionsAreSubset(RequestInfo requestInfo, AuthorizationInfo authorizationInfo,
+                                                  Map<String, List<String>> indexNameToNewNames,
+                                                  ActionListener<AuthorizationResult> listener) {
+        if (isSuperuser(requestInfo.getAuthentication().getUser())) {
+            listener.onResponse(AuthorizationResult.granted());
+        } else {
+            listener.onResponse(AuthorizationResult.deny());
+        }
+    }
+
+    @Override
+    public void checkPrivileges(Authentication authentication, AuthorizationInfo authorizationInfo,
+                                HasPrivilegesRequest hasPrivilegesRequest,
+                                Collection<ApplicationPrivilegeDescriptor> applicationPrivilegeDescriptors,
+                                ActionListener<HasPrivilegesResponse> listener) {
+        if (isSuperuser(authentication.getUser())) {
+            listener.onResponse(getHasPrivilegesResponse(authentication, hasPrivilegesRequest, true));
+        } else {
+            listener.onResponse(getHasPrivilegesResponse(authentication, hasPrivilegesRequest, false));
+        }
+    }
+
+    @Override
+    public void getUserPrivileges(Authentication authentication, AuthorizationInfo authorizationInfo, GetUserPrivilegesRequest request,
+                                  ActionListener<GetUserPrivilegesResponse> listener) {
+        if (isSuperuser(authentication.getUser())) {
+            listener.onResponse(getUserPrivilegesResponse(true));
+        } else {
+            listener.onResponse(getUserPrivilegesResponse(false));
+        }
+    }
+
+    private HasPrivilegesResponse getHasPrivilegesResponse(Authentication authentication, HasPrivilegesRequest hasPrivilegesRequest,
+                                                           boolean authorized) {
+        Map<String, Boolean> clusterPrivMap = new HashMap<>();
+        for (String clusterPriv : hasPrivilegesRequest.clusterPrivileges()) {
+            clusterPrivMap.put(clusterPriv, authorized);
+        }
+        final Map<String, ResourcePrivileges> indices = new LinkedHashMap<>();
+        for (IndicesPrivileges check : hasPrivilegesRequest.indexPrivileges()) {
+            for (String index : check.getIndices()) {
+                final Map<String, Boolean> privileges = new HashMap<>();
+                final ResourcePrivileges existing = indices.get(index);
+                if (existing != null) {
+                    privileges.putAll(existing.getPrivileges());
+                }
+                for (String privilege : check.getPrivileges()) {
+                    privileges.put(privilege, authorized);
+                }
+                indices.put(index, ResourcePrivileges.builder(index).addPrivileges(privileges).build());
+            }
+        }
+        final Map<String, Collection<ResourcePrivileges>> privilegesByApplication = new HashMap<>();
+        Set<String> applicationNames = Arrays.stream(hasPrivilegesRequest.applicationPrivileges())
+            .map(RoleDescriptor.ApplicationResourcePrivileges::getApplication)
+            .collect(Collectors.toSet());
+        for (String applicationName : applicationNames) {
+            final Map<String, ResourcePrivileges> appPrivilegesByResource = new LinkedHashMap<>();
+            for (RoleDescriptor.ApplicationResourcePrivileges p : hasPrivilegesRequest.applicationPrivileges()) {
+                if (applicationName.equals(p.getApplication())) {
+                    for (String resource : p.getResources()) {
+                        final Map<String, Boolean> privileges = new HashMap<>();
+                        final ResourcePrivileges existing = appPrivilegesByResource.get(resource);
+                        if (existing != null) {
+                            privileges.putAll(existing.getPrivileges());
+                        }
+                        for (String privilege : p.getPrivileges()) {
+                            privileges.put(privilege, authorized);
+                        }
+                        appPrivilegesByResource.put(resource, ResourcePrivileges.builder(resource).addPrivileges(privileges).build());
+                    }
+                }
+            }
+            privilegesByApplication.put(applicationName, appPrivilegesByResource.values());
+        }
+        return new HasPrivilegesResponse(authentication.getUser().principal(), authorized, clusterPrivMap, indices.values(),
+            privilegesByApplication);
+    }
+
+    private GetUserPrivilegesResponse getUserPrivilegesResponse(boolean isSuperuser) {
+        final Set<String> cluster = isSuperuser ? Collections.singleton("ALL") : Collections.emptySet();
+        final Set<ConditionalClusterPrivilege> conditionalCluster = Collections.emptySet();
+        final Set<GetUserPrivilegesResponse.Indices> indices = isSuperuser ? Collections.singleton(new Indices(Collections.singleton("*"),
+            Collections.singleton("*"), Collections.emptySet(), Collections.emptySet(), true)) : Collections.emptySet();
+
+        final Set<RoleDescriptor.ApplicationResourcePrivileges> application = isSuperuser ?
+            Collections.singleton(
+                RoleDescriptor.ApplicationResourcePrivileges.builder().application("*").privileges("*").resources("*").build()) :
+            Collections.emptySet();
+        final Set<String> runAs = isSuperuser ? Collections.singleton("*") : Collections.emptySet();
+        return new GetUserPrivilegesResponse(cluster, conditionalCluster, indices, application, runAs);
+    }
+
+    public static class CustomAuthorizationInfo implements AuthorizationInfo {
+
+        private final String[] roles;
+        private final CustomAuthorizationInfo authenticatedAuthzInfo;
+
+        CustomAuthorizationInfo(String[] roles, CustomAuthorizationInfo authenticatedAuthzInfo) {
+            this.roles = roles;
+            this.authenticatedAuthzInfo = authenticatedAuthzInfo;
+        }
+
+        @Override
+        public Map<String, Object> asMap() {
+            return Collections.singletonMap("roles", roles);
+        }
+
+        @Override
+        public CustomAuthorizationInfo getAuthenticatedUserAuthorizationInfo() {
+            return authenticatedAuthzInfo;
+        }
+    }
+
+    private boolean isSuperuser(User user) {
+        return Arrays.asList(user.roles()).contains("custom_superuser");
+    }
+}

plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/ExampleAuthorizationEngineExtension.java

@@ -0,0 +1,35 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.example;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xpack.core.security.SecurityExtension;
+import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
+
+/**
+ * Security extension class that registers the custom authorization engine to be used
+ */
+public class ExampleAuthorizationEngineExtension implements SecurityExtension {
+
+    @Override
+    public AuthorizationEngine getAuthorizationEngine(Settings settings) {
+        return new CustomAuthorizationEngine();
+    }
+}

plugins/examples/security-authorization-engine/src/main/resources/META-INF/services/org.elasticsearch.xpack.core.security.SecurityExtension

@@ -0,0 +1 @@
+org.elasticsearch.example.ExampleAuthorizationEngineExtension