Authorize rest requests

build.gradle

@@ -289,7 +289,7 @@ configurations {
             force "io.netty:netty-transport:${versions.netty}"
             force "io.netty:netty-transport-native-unix-common:${versions.netty}"
             force "org.apache.bcel:bcel:6.6.0" // This line should be removed once Spotbugs is upgraded to 4.7.4
-            force "com.github.luben:zstd-jni:${versions.zstd}"
+            force "com.github.luben:zstd-jni:1.5.5-3"
         }
     }
 
@@ -412,7 +412,7 @@ dependencies {
     runtimeOnly 'com.fasterxml.woodstox:woodstox-core:6.4.0'
     runtimeOnly 'org.apache.ws.xmlschema:xmlschema-core:2.2.5'
     runtimeOnly 'org.apache.santuario:xmlsec:2.2.3'
-    runtimeOnly "com.github.luben:zstd-jni:${versions.zstd}"
+    runtimeOnly "com.github.luben:zstd-jni:1.5.5-3"
     runtimeOnly 'org.checkerframework:checker-qual:3.5.0'
     runtimeOnly "org.bouncycastle:bcpkix-jdk15on:${versions.bouncycastle}"
 

config/internal_users.yml

@@ -61,3 +61,8 @@ snapshotrestore:
   backend_roles:
   - "snapshotrestore"
   description: "Demo snapshotrestore user, using external role mapping"
+
+new-user:
+  hash: "$2a$12$VcCDgh2NDk07JGN0rjGbM.Ad41qVR/YFJcgHp0UGns5JDymv..TOG"
+  reserved: true
+  description: "Demo user for ext-test"

config/roles.yml

@@ -355,3 +355,22 @@ security_analytics_ack_alerts:
   reserved: true
   cluster_permissions:
     - 'cluster:admin/opensearch/securityanalytics/alerts/*'
+
+# Roles for Hello World extension, roles that extensions would like to add should be ultimately be defined elsewhere
+extension_hw_greet:
+  reserved: true
+  cluster_permissions:
+    - 'hw:greet'
+
+legacy_hw_greet_with_name:
+  reserved: true
+  cluster_permissions:
+    - 'cluster:admin/opensearch/hw/greet_with_name'
+
+extension_hw_full:
+  reserved: true
+  cluster_permissions:
+    - 'hw:greet'
+    - 'hw:greet_with_adjective'
+    - 'hw:great_with_name'
+    - 'hw:goodbye'

config/roles_mapping.yml

@@ -47,3 +47,14 @@ kibana_server:
   reserved: true
   users:
   - "kibanaserver"
+
+## Demo test roles
+legacy_hw_greet_with_name:
+  reserved: true
+  users:
+    - "new-user"
+
+extension_hw_greet:
+  reserved: true
+  users:
+    - "new-user"

src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java

@@ -146,6 +146,7 @@
 import org.opensearch.security.http.XFFResolver;
 import org.opensearch.security.privileges.PrivilegesEvaluator;
 import org.opensearch.security.privileges.PrivilegesInterceptor;
+import org.opensearch.security.privileges.RestLayerPrivilegesEvaluator;
 import org.opensearch.security.resolver.IndexResolverReplacer;
 import org.opensearch.security.rest.DashboardsInfoAction;
 import org.opensearch.security.rest.SecurityConfigUpdateAction;
@@ -205,6 +206,7 @@ public final class OpenSearchSecurityPlugin extends OpenSearchSecuritySSLPlugin
     private volatile SecurityInterceptor si;
     private volatile PrivilegesEvaluator evaluator;
     private volatile UserService userService;
+    private volatile RestLayerPrivilegesEvaluator restLayerEvaluator;
     private volatile ThreadPool threadPool;
     private volatile ConfigurationRepository cr;
     private volatile AdminDNs adminDns;
@@ -1019,8 +1021,11 @@ public Collection<Object> createComponents(
             principalExtractor = ReflectionHelper.instantiatePrincipalExtractor(principalExtractorClass);
         }
 
+        restLayerEvaluator = new RestLayerPrivilegesEvaluator(clusterService, threadPool, auditLog, cih, namedXContentRegistry);
+
         securityRestHandler = new SecurityRestFilter(
             backendRegistry,
+            restLayerEvaluator,
             auditLog,
             threadPool,
             principalExtractor,
@@ -1035,6 +1040,7 @@ public Collection<Object> createComponents(
         dcf.registerDCFListener(irr);
         dcf.registerDCFListener(xffResolver);
         dcf.registerDCFListener(evaluator);
+        dcf.registerDCFListener(restLayerEvaluator);
         dcf.registerDCFListener(securityRestHandler);
         if (!(auditLog instanceof NullAuditLog)) {
             // Don't register if advanced modules is disabled in which case auditlog is instance of NullAuditLog
@@ -1072,6 +1078,7 @@ public Collection<Object> createComponents(
         components.add(xffResolver);
         components.add(backendRegistry);
         components.add(evaluator);
+        components.add(restLayerEvaluator);
         components.add(si);
         components.add(dcf);
         components.add(userService);

src/main/java/org/opensearch/security/filter/SecurityRestFilter.java

@@ -27,6 +27,9 @@
 package org.opensearch.security.filter;
 
 import java.nio.file.Path;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -37,10 +40,10 @@
 import org.greenrobot.eventbus.Subscribe;
 
 import org.opensearch.OpenSearchException;
-import org.opensearch.client.node.NodeClient;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.rest.BytesRestResponse;
+import org.opensearch.rest.NamedRoute;
 import org.opensearch.rest.RestChannel;
 import org.opensearch.rest.RestHandler;
 import org.opensearch.rest.RestRequest;
@@ -52,6 +55,8 @@
 import org.opensearch.security.configuration.AdminDNs;
 import org.opensearch.security.configuration.CompatConfig;
 import org.opensearch.security.dlic.rest.api.AllowlistApiAction;
+import org.opensearch.security.privileges.PrivilegesEvaluatorResponse;
+import org.opensearch.security.privileges.RestLayerPrivilegesEvaluator;
 import org.opensearch.security.securityconf.impl.AllowlistingSettings;
 import org.opensearch.security.securityconf.impl.WhitelistingSettings;
 import org.opensearch.security.ssl.transport.PrincipalExtractor;
@@ -70,6 +75,7 @@ public class SecurityRestFilter {
 
     protected final Logger log = LogManager.getLogger(this.getClass());
     private final BackendRegistry registry;
+    private final RestLayerPrivilegesEvaluator evaluator;
     private final AuditLog auditLog;
     private final ThreadContext threadContext;
     private final PrincipalExtractor principalExtractor;
@@ -88,6 +94,7 @@ public class SecurityRestFilter {
 
     public SecurityRestFilter(
         final BackendRegistry registry,
+        final RestLayerPrivilegesEvaluator evaluator,
         final AuditLog auditLog,
         final ThreadPool threadPool,
         final PrincipalExtractor principalExtractor,
@@ -97,6 +104,7 @@ public SecurityRestFilter(
     ) {
         super();
         this.registry = registry;
+        this.evaluator = evaluator;
         this.auditLog = auditLog;
         this.threadContext = threadPool.getThreadContext();
         this.principalExtractor = principalExtractor;
@@ -109,28 +117,26 @@ public SecurityRestFilter(
 
     /**
      * This function wraps around all rest requests
-     * If the request is authenticated, then it goes through a whitelisting check.
-     * The whitelisting check works as follows:
-     * If whitelisting is not enabled, then requests are handled normally.
-     * If whitelisting is enabled, then SuperAdmin is allowed access to all APIs, regardless of what is currently whitelisted.
-     * If whitelisting is enabled, then Non-SuperAdmin is allowed to access only those APIs that are whitelisted in {@link #requests}
-     * For example: if whitelisting is enabled and requests = ["/_cat/nodes"], then SuperAdmin can access all APIs, but non SuperAdmin
+     * If the request is authenticated, then it goes through a allowlisting check.
+     * The allowlisting check works as follows:
+     * If allowlisting is not enabled, then requests are handled normally.
+     * If allowlisting is enabled, then SuperAdmin is allowed access to all APIs, regardless of what is currently allowlisted.
+     * If allowlisting is enabled, then Non-SuperAdmin is allowed to access only those APIs that are allowlisted in {@link #requests}
+     * For example: if allowlisting is enabled and requests = ["/_cat/nodes"], then SuperAdmin can access all APIs, but non SuperAdmin
      * can only access "/_cat/nodes"
-     * Further note: Some APIs are only accessible by SuperAdmin, regardless of whitelisting. For example: /_opendistro/_security/api/whitelist is only accessible by SuperAdmin.
+     * Further note: Some APIs are only accessible by SuperAdmin, regardless of allowlisting. For example: /_opendistro/_security/api/whitelist is only accessible by SuperAdmin.
      * See {@link AllowlistApiAction} for the implementation of this API.
      * SuperAdmin is identified by credentials, which can be passed in the curl request.
      */
     public RestHandler wrap(RestHandler original, AdminDNs adminDNs) {
-        return new RestHandler() {
-
-            @Override
-            public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
-                org.apache.logging.log4j.ThreadContext.clearAll();
-                if (!checkAndAuthenticateRequest(request, channel, client)) {
-                    User user = threadContext.getTransient(ConfigConstants.OPENDISTRO_SECURITY_USER);
-                    if (userIsSuperAdmin(user, adminDNs)
-                        || (whitelistingSettings.checkRequestIsAllowed(request, channel, client)
-                            && allowlistingSettings.checkRequestIsAllowed(request, channel, client))) {
+        return (request, channel, client) -> {
+            org.apache.logging.log4j.ThreadContext.clearAll();
+            if (!checkAndAuthenticateRequest(request, channel)) {
+                User user = threadContext.getTransient(ConfigConstants.OPENDISTRO_SECURITY_USER);
+                if (userIsSuperAdmin(user, adminDNs)
+                    || (whitelistingSettings.checkRequestIsAllowed(request, channel, client)
+                        && allowlistingSettings.checkRequestIsAllowed(request, channel, client))) {
+                    if (authorizeRequest(original, request, channel, user)) {
                         original.handleRequest(request, channel, client);
                     }
                 }
@@ -145,7 +151,56 @@ private boolean userIsSuperAdmin(User user, AdminDNs adminDNs) {
         return user != null && adminDNs.isAdmin(user);
     }
 
-    private boolean checkAndAuthenticateRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
+    private boolean authorizeRequest(RestHandler original, RestRequest request, RestChannel channel, User user) {
+
+        List<RestHandler.Route> restRoutes = original.routes();
+        Optional<RestHandler.Route> handler = restRoutes.stream()
+            .filter(rh -> rh.getMethod().equals(request.method()))
+            .filter(rh -> restPathMatches(request.path(), rh.getPath()))
+            .findFirst();
+        if (handler.isPresent() && handler.get() instanceof NamedRoute) {
+            PrivilegesEvaluatorResponse pres = new PrivilegesEvaluatorResponse();
+            NamedRoute route = ((NamedRoute) handler.get());
+            // if actionNames are present evaluate those first
+            Set<String> actionNames = route.actionNames();
+            if (actionNames != null && !actionNames.isEmpty()) {
+                pres = evaluator.evaluate(user, actionNames);
+            }
+
+            // now if pres.allowed is still false check for the NamedRoute name as a permission
+            if (!pres.isAllowed()) {
+                String action = route.name();
+                pres = evaluator.evaluate(user, Set.of(action));
+            }
+
+            if (log.isDebugEnabled()) {
+                log.debug(pres.toString());
+            }
+            if (pres.isAllowed()) {
+                // TODO make sure this is audit logged
+                log.debug("Request has been granted");
+                // auditLog.logGrantedPrivileges(action, request, task);
+            } else {
+                // auditLog.logMissingPrivileges(action, request, task);
+                String err;
+                if (!pres.getMissingSecurityRoles().isEmpty()) {
+                    err = String.format("No mapping for %s on roles %s", user, pres.getMissingSecurityRoles());
+                } else {
+                    err = String.format("no permissions for %s and %s", pres.getMissingPrivileges(), user);
+                }
+                log.debug(err);
+                // TODO Figure out why ext hangs intermittently after single unauthorized request
+                channel.sendResponse(new BytesRestResponse(RestStatus.UNAUTHORIZED, err));
+                return false;
+            }
+        }
+
+        // if handler is not an instance of NamedRoute then we pass through to eval at Transport Layer.
+        // TODO Change this to false once all plugins have migrated to use NamedRoutes
+        return true;
+    }
+
+    private boolean checkAndAuthenticateRequest(RestRequest request, RestChannel channel) throws Exception {
 
         threadContext.putTransient(ConfigConstants.OPENDISTRO_SECURITY_ORIGIN, Origin.REST.toString());
 
@@ -217,4 +272,30 @@ public void onWhitelistingSettingChanged(WhitelistingSettings whitelistingSettin
     public void onAllowlistingSettingChanged(AllowlistingSettings allowlistingSettings) {
         this.allowlistingSettings = allowlistingSettings;
     }
+
+    /**
+     * Determines if the request's path is a match for the configured handler path.
+     *
+     * @param requestPath The path from the {@link NamedRoute}
+     * @param handlerPath The path from the {@link RestHandler.Route}
+     * @return true if the request path matches the route
+     */
+    private boolean restPathMatches(String requestPath, String handlerPath) {
+        // Check exact match
+        if (handlerPath.equals(requestPath)) {
+            return true;
+        }
+        // Split path to evaluate named params
+        String[] handlerSplit = handlerPath.split("/");
+        String[] requestSplit = requestPath.split("/");
+        if (handlerSplit.length != requestSplit.length) {
+            return false;
+        }
+        for (int i = 0; i < handlerSplit.length; i++) {
+            if (!(handlerSplit[i].equals(requestSplit[i]) || (handlerSplit[i].startsWith("{") && handlerSplit[i].endsWith("}")))) {
+                return false;
+            }
+        }
+        return true;
+    }
 }

src/main/java/org/opensearch/security/privileges/PrivilegesEvaluator.java

@@ -338,7 +338,10 @@ public PrivilegesEvaluatorResponse evaluate(
         );
 
         if (isClusterPerm(action0)) {
-            if (!securityRoles.impliesClusterPermissionPermission(action0)) {
+            // We add a check here for `impliesLegacyPermission` to ensure that no new action names miss evaluation here
+            // e.g. ad:get/detector/info will be evaluated as `cluster:admin/opensearch/ad/get/detector/info` against user's granted
+            // permissions
+            if (!securityRoles.impliesClusterPermissionPermission(action0) && !securityRoles.impliesLegacyPermission(action0)) {
                 presponse.missingPrivileges.add(action0);
                 presponse.allowed = false;
                 log.info(