Introduce Application Privileges to Roles (#30164)

This commit introduces "Application Privileges" (aka custom privileges) to the X-Pack security model.

Application Privileges are managed within Elasticsearch, and can be tested with the _has_privileges API, but do not grant access to any actions or resources within Elasticsearch.
Their purpose is to allow applications outside of Elasticsearch to represent and store their own privileges model within Elasticsearch roles.

Specifically, this adds
- GET/PUT/DELETE actions for defining application level privileges
- application privileges in role definitions
- application privileges in the has_privileges API

server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java

@@ -59,6 +59,7 @@
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
 import java.util.EnumSet;
@@ -70,6 +71,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 import java.util.function.IntFunction;
 import java.util.function.Supplier;
 
@@ -932,8 +934,23 @@ public <T extends Streamable> List<T> readStreamableList(Supplier<T> constructor
      * Reads a list of objects
      */
     public <T> List<T> readList(Writeable.Reader<T> reader) throws IOException {
+        return readCollection(reader, ArrayList::new);
+    }
+
+    /**
+     * Reads a set of objects
+     */
+    public <T> Set<T> readSet(Writeable.Reader<T> reader) throws IOException {
+        return readCollection(reader, HashSet::new);
+    }
+
+    /**
+     * Reads a collection of objects
+     */
+    private <T, C extends Collection<? super T>> C readCollection(Writeable.Reader<T> reader,
+                                                                  IntFunction<C> constructor) throws IOException {
         int count = readArraySize();
-        List<T> builder = new ArrayList<>(count);
+        C builder = constructor.apply(count);
         for (int i=0; i<count; i++) {
             builder.add(reader.read(this));
         }

server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java

@@ -55,6 +55,7 @@
 import java.nio.file.NoSuchFileException;
 import java.nio.file.NotDirectoryException;
 import java.time.ZonedDateTime;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
 import java.util.EnumMap;
@@ -995,6 +996,16 @@ public void writeList(List<? extends Writeable> list) throws IOException {
         }
     }
 
+    /**
+     * Writes a collection of generic objects via a {@link Writer}
+     */
+    public <T> void writeCollection(Collection<T> collection, Writer<T> writer) throws IOException {
+        writeVInt(collection.size());
+        for (T val: collection) {
+            writer.write(this, val);
+        }
+    }
+
     /**
      * Writes a list of strings
      */

server/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java

@@ -31,6 +31,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
@@ -42,6 +43,7 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasToString;
+import static org.hamcrest.Matchers.iterableWithSize;
 
 public class StreamTests extends ESTestCase {
 
@@ -65,7 +67,7 @@ public void testBooleanSerialization() throws IOException {
         final Set<Byte> set = IntStream.range(Byte.MIN_VALUE, Byte.MAX_VALUE).mapToObj(v -> (byte) v).collect(Collectors.toSet());
         set.remove((byte) 0);
         set.remove((byte) 1);
-        final byte[] corruptBytes = new byte[] { randomFrom(set) };
+        final byte[] corruptBytes = new byte[]{randomFrom(set)};
         final BytesReference corrupt = new BytesArray(corruptBytes);
         final IllegalStateException e = expectThrows(IllegalStateException.class, () -> corrupt.streamInput().readBoolean());
         final String message = String.format(Locale.ROOT, "unexpected byte [0x%02x]", corruptBytes[0]);
@@ -100,7 +102,7 @@ public void testOptionalBooleanSerialization() throws IOException {
         set.remove((byte) 0);
         set.remove((byte) 1);
         set.remove((byte) 2);
-        final byte[] corruptBytes = new byte[] { randomFrom(set) };
+        final byte[] corruptBytes = new byte[]{randomFrom(set)};
         final BytesReference corrupt = new BytesArray(corruptBytes);
         final IllegalStateException e = expectThrows(IllegalStateException.class, () -> corrupt.streamInput().readOptionalBoolean());
         final String message = String.format(Locale.ROOT, "unexpected byte [0x%02x]", corruptBytes[0]);
@@ -119,22 +121,22 @@ public void testRandomVLongSerialization() throws IOException {
 
     public void testSpecificVLongSerialization() throws IOException {
         List<Tuple<Long, byte[]>> values =
-                Arrays.asList(
-                        new Tuple<>(0L, new byte[]{0}),
-                        new Tuple<>(-1L, new byte[]{1}),
-                        new Tuple<>(1L, new byte[]{2}),
-                        new Tuple<>(-2L, new byte[]{3}),
-                        new Tuple<>(2L, new byte[]{4}),
-                        new Tuple<>(Long.MIN_VALUE, new byte[]{-1, -1, -1, -1, -1, -1, -1, -1, -1, 1}),
-                        new Tuple<>(Long.MAX_VALUE, new byte[]{-2, -1, -1, -1, -1, -1, -1, -1, -1, 1})
-
-                );
+            Arrays.asList(
+                new Tuple<>(0L, new byte[]{0}),
+                new Tuple<>(-1L, new byte[]{1}),
+                new Tuple<>(1L, new byte[]{2}),
+                new Tuple<>(-2L, new byte[]{3}),
+                new Tuple<>(2L, new byte[]{4}),
+                new Tuple<>(Long.MIN_VALUE, new byte[]{-1, -1, -1, -1, -1, -1, -1, -1, -1, 1}),
+                new Tuple<>(Long.MAX_VALUE, new byte[]{-2, -1, -1, -1, -1, -1, -1, -1, -1, 1})
+
+            );
         for (Tuple<Long, byte[]> value : values) {
             BytesStreamOutput out = new BytesStreamOutput();
             out.writeZLong(value.v1());
             assertArrayEquals(Long.toString(value.v1()), value.v2(), BytesReference.toBytes(out.bytes()));
             BytesReference bytes = new BytesArray(value.v2());
-            assertEquals(Arrays.toString(value.v2()), (long)value.v1(), bytes.streamInput().readZLong());
+            assertEquals(Arrays.toString(value.v2()), (long) value.v1(), bytes.streamInput().readZLong());
         }
     }
 
@@ -158,7 +160,7 @@ public void testLinkedHashMap() throws IOException {
         }
         BytesStreamOutput out = new BytesStreamOutput();
         out.writeGenericValue(write);
-        LinkedHashMap<String, Integer> read = (LinkedHashMap<String, Integer>)out.bytes().streamInput().readGenericValue();
+        LinkedHashMap<String, Integer> read = (LinkedHashMap<String, Integer>) out.bytes().streamInput().readGenericValue();
         assertEquals(size, read.size());
         int index = 0;
         for (Map.Entry<String, Integer> entry : read.entrySet()) {
@@ -172,7 +174,8 @@ public void testFilterStreamInputDelegatesAvailable() throws IOException {
         final int length = randomIntBetween(1, 1024);
         StreamInput delegate = StreamInput.wrap(new byte[length]);
 
-        FilterStreamInput filterInputStream = new FilterStreamInput(delegate) {};
+        FilterStreamInput filterInputStream = new FilterStreamInput(delegate) {
+        };
         assertEquals(filterInputStream.available(), length);
 
         // read some bytes
@@ -201,7 +204,7 @@ public void testReadArraySize() throws IOException {
         }
         stream.writeByteArray(array);
         InputStreamStreamInput streamInput = new InputStreamStreamInput(StreamInput.wrap(BytesReference.toBytes(stream.bytes())), array
-            .length-1);
+            .length - 1);
         expectThrows(EOFException.class, streamInput::readByteArray);
         streamInput = new InputStreamStreamInput(StreamInput.wrap(BytesReference.toBytes(stream.bytes())), BytesReference.toBytes(stream
             .bytes()).length);
@@ -230,6 +233,21 @@ public void testWritableArrays() throws IOException {
         assertThat(targetArray, equalTo(sourceArray));
     }
 
+    public void testSetOfLongs() throws IOException {
+        final int size = randomIntBetween(0, 6);
+        final Set<Long> sourceSet = new HashSet<>(size);
+        for (int i = 0; i < size; i++) {
+            sourceSet.add(randomLongBetween(i * 1000, (i + 1) * 1000 - 1));
+        }
+        assertThat(sourceSet, iterableWithSize(size));
+
+        final BytesStreamOutput out = new BytesStreamOutput();
+        out.writeCollection(sourceSet, StreamOutput::writeLong);
+
+        final Set<Long> targetSet = out.bytes().streamInput().readSet(StreamInput::readLong);
+        assertThat(targetSet, equalTo(sourceSet));
+    }
+
     static final class WriteableString implements Writeable {
         final String string;
 

test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java

@@ -141,6 +141,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BooleanSupplier;
 import java.util.function.Consumer;
+import java.util.function.IntFunction;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
@@ -675,6 +676,16 @@ public static String[] generateRandomStringArray(int maxArraySize, int stringSiz
         return generateRandomStringArray(maxArraySize, stringSize, allowNull, true);
     }
 
+    public static <T> T[] randomArray(int maxArraySize, IntFunction<T[]> arrayConstructor, Supplier<T> valueConstructor) {
+        final int size = randomInt(maxArraySize);
+        final T[] array = arrayConstructor.apply(size);
+        for (int i = 0; i < array.length; i++) {
+            array[i] = valueConstructor.get();
+        }
+        return array;
+    }
+
+
     private static final String[] TIME_SUFFIXES = new String[]{"d", "h", "ms", "s", "m", "micros", "nanos"};
 
     public static String randomTimeValue(int lower, int upper, String... suffixes) {

x-pack/docs/en/rest-api/security/privileges.asciidoc

@@ -84,7 +84,8 @@ The following example output indicates which privileges the "rdeniro" user has:
       "read" : true,
       "write" : false
     }
-  }
+  },
+  "application" : {}
 }
 --------------------------------------------------
 // TESTRESPONSE[s/"rdeniro"/"$body.username"/]

x-pack/docs/en/rest-api/security/roles.asciidoc

@@ -138,6 +138,7 @@ role. If the role is not defined in the `native` realm, the request 404s.
       },
       "query" : "{\"match\": {\"title\": \"foo\"}}"
     } ],
+    "applications" : [ ],
     "run_as" : [ "other_user" ],
     "metadata" : {
       "version" : 1

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesAction.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.client.ElasticsearchClient;
+
+/**
+ * Action for deleting application privileges.
+ */
+public final class DeletePrivilegesAction extends Action<DeletePrivilegesRequest, DeletePrivilegesResponse> {
+
+    public static final DeletePrivilegesAction INSTANCE = new DeletePrivilegesAction();
+    public static final String NAME = "cluster:admin/xpack/security/privilege/delete";
+
+    private DeletePrivilegesAction() {
+        super(NAME);
+    }
+
+    @Override
+    public DeletePrivilegesResponse newResponse() {
+        return new DeletePrivilegesResponse();
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesRequest.java

@@ -0,0 +1,93 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * A request to delete an application privilege.
+ */
+public final class DeletePrivilegesRequest extends ActionRequest implements WriteRequest<DeletePrivilegesRequest> {
+
+    private String application;
+    private String[] privileges;
+    private RefreshPolicy refreshPolicy = RefreshPolicy.IMMEDIATE;
+
+    public DeletePrivilegesRequest() {
+        this(null, Strings.EMPTY_ARRAY);
+    }
+
+    public DeletePrivilegesRequest(String application, String[] privileges) {
+        this.application = application;
+        this.privileges = privileges;
+    }
+
+    @Override
+    public DeletePrivilegesRequest setRefreshPolicy(RefreshPolicy refreshPolicy) {
+        this.refreshPolicy = refreshPolicy;
+        return this;
+    }
+
+    @Override
+    public RefreshPolicy getRefreshPolicy() {
+        return refreshPolicy;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (Strings.isNullOrEmpty(application)) {
+            validationException = addValidationError("application name is missing", validationException);
+        }
+        if (privileges == null || privileges.length == 0 || Arrays.stream(privileges).allMatch(Strings::isNullOrEmpty)) {
+            validationException = addValidationError("privileges are missing", validationException);
+        }
+        return validationException;
+    }
+
+    public void application(String application) {
+        this.application = application;
+    }
+
+    public String application() {
+        return application;
+    }
+
+    public String[] privileges() {
+        return this.privileges;
+    }
+
+    public void privileges(String[] privileges) {
+        this.privileges = privileges;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        application = in.readString();
+        privileges = in.readStringArray();
+        refreshPolicy = RefreshPolicy.readFrom(in);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(application);
+        out.writeStringArray(privileges);
+        refreshPolicy.writeTo(out);
+    }
+
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/DeletePrivilegesRequestBuilder.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.core.security.action.privilege;
+
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.support.WriteRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+
+import java.util.Collection;
+
+/**
+ * Builder for {@link DeletePrivilegesRequest}
+ */
+public final class DeletePrivilegesRequestBuilder     extends ActionRequestBuilder<DeletePrivilegesRequest, DeletePrivilegesResponse>
+        implements WriteRequestBuilder<DeletePrivilegesRequestBuilder> {
+
+    public DeletePrivilegesRequestBuilder(ElasticsearchClient client, DeletePrivilegesAction action) {
+        super(client, action, new DeletePrivilegesRequest());
+    }
+
+    public DeletePrivilegesRequestBuilder privileges(String[] privileges) {
+        request.privileges(privileges);
+        return this;
+    }
+
+    public DeletePrivilegesRequestBuilder application(String applicationName) {
+        request.application(applicationName);
+        return this;
+    }
+}