[Kerberos] Add Kerberos authentication support (#32263)

This commit adds support for Kerberos authentication with a platinum
license. Kerberos authentication support relies on SPNEGO, which is
triggered by challenging clients with a 401 response with the
`WWW-Authenticate: Negotiate` header. A SPNEGO client will then provide
a Kerberos ticket in the `Authorization` header. The tickets are
validated using Java's built-in GSS support. The JVM uses a vm wide
configuration for Kerberos, so there can be only one Kerberos realm.
This is enforced by a bootstrap check that also enforces the existence
of the keytab file.

In many cases a fallback authentication mechanism is needed when SPNEGO
authentication is not available. In order to support this, the
DefaultAuthenticationFailureHandler now takes a list of failure response
headers. For example, one realm can provide a
`WWW-Authenticate: Negotiate` header as its default and another could
provide `WWW-Authenticate: Basic` to indicate to the client that basic
authentication can be used in place of SPNEGO.

In order to test Kerberos, unit tests are run against an in-memory KDC
that is backed by an in-memory ldap server. A QA project has also been
added to test against an actual KDC, which is provided by the krb5kdc
fixture.

Closes #30243

test/fixtures/krb5kdc-fixture/src/main/resources/provision/addprinc.sh

@@ -20,11 +20,14 @@
 set -e
 
 if [[ $# -lt 1 ]]; then
-  echo 'Usage: addprinc.sh <principalNameNoRealm>'
+  echo 'Usage: addprinc.sh principalName [password]'
+  echo '  principalName    user principal name without realm'
+  echo '  password         If provided then will set password for user else it will provision user with keytab'
   exit 1
 fi
 
 PRINC="$1"
+PASSWD="$2"
 USER=$(echo $PRINC | tr "/" "_")
 
 VDIR=/vagrant
@@ -47,12 +50,17 @@ ADMIN_KTAB=$LOCALSTATEDIR/admin.keytab
 USER_PRIN=$PRINC@$REALM
 USER_KTAB=$LOCALSTATEDIR/$USER.keytab
 
-if [ -f $USER_KTAB ]; then
+if [ -f $USER_KTAB ] && [ -z "$PASSWD" ]; then
   echo "Principal '${PRINC}@${REALM}' already exists. Re-copying keytab..."
+  sudo cp $USER_KTAB $KEYTAB_DIR/$USER.keytab
 else
-  echo "Provisioning '${PRINC}@${REALM}' principal and keytab..."
-  sudo kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "addprinc -randkey $USER_PRIN"
-  sudo kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "ktadd -k $USER_KTAB $USER_PRIN"
+  if [ -z "$PASSWD" ]; then
+    echo "Provisioning '${PRINC}@${REALM}' principal and keytab..."
+    sudo kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "addprinc -randkey $USER_PRIN"
+    sudo kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "ktadd -k $USER_KTAB $USER_PRIN"
+    sudo cp $USER_KTAB $KEYTAB_DIR/$USER.keytab
+  else
+    echo "Provisioning '${PRINC}@${REALM}' principal with password..."
+    sudo kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "addprinc -pw $PASSWD $PRINC"
+  fi
 fi
-
-sudo cp $USER_KTAB $KEYTAB_DIR/$USER.keytab

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandler.java

@@ -10,60 +10,132 @@
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.transport.TransportMessage;
+import org.elasticsearch.xpack.core.XPackField;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
 
 import static org.elasticsearch.xpack.core.security.support.Exceptions.authenticationError;
 
 /**
- * The default implementation of a {@link AuthenticationFailureHandler}. This handler will return an exception with a
- * RestStatus of 401 and the WWW-Authenticate header with a Basic challenge.
+ * The default implementation of a {@link AuthenticationFailureHandler}. This
+ * handler will return an exception with a RestStatus of 401 and default failure
+ * response headers like 'WWW-Authenticate'
  */
 public class DefaultAuthenticationFailureHandler implements AuthenticationFailureHandler {
+    private final Map<String, List<String>> defaultFailureResponseHeaders;
+
+    /**
+     * Constructs default authentication failure handler
+     *
+     * @deprecated replaced by {@link #DefaultAuthenticationFailureHandler(Map)}
+     */
+    @Deprecated
+    public DefaultAuthenticationFailureHandler() {
+        this(null);
+    }
+
+    /**
+     * Constructs default authentication failure handler with provided default
+     * response headers.
+     *
+     * @param failureResponseHeaders Map of header key and list of header values to
+     *            be sent as failure response.
+     * @see Realm#getAuthenticationFailureHeaders()
+     */
+    public DefaultAuthenticationFailureHandler(Map<String, List<String>> failureResponseHeaders) {
+        if (failureResponseHeaders == null || failureResponseHeaders.isEmpty()) {
+            failureResponseHeaders = Collections.singletonMap("WWW-Authenticate",
+                    Collections.singletonList("Basic realm=\"" + XPackField.SECURITY + "\" charset=\"UTF-8\""));
+        }
+        this.defaultFailureResponseHeaders = Collections.unmodifiableMap(failureResponseHeaders);
+    }
 
     @Override
-    public ElasticsearchSecurityException failedAuthentication(RestRequest request, AuthenticationToken token,
-                                                               ThreadContext context) {
-        return authenticationError("unable to authenticate user [{}] for REST request [{}]", token.principal(), request.uri());
+    public ElasticsearchSecurityException failedAuthentication(RestRequest request, AuthenticationToken token, ThreadContext context) {
+        return createAuthenticationError("unable to authenticate user [{}] for REST request [{}]", null, token.principal(), request.uri());
     }
 
     @Override
     public ElasticsearchSecurityException failedAuthentication(TransportMessage message, AuthenticationToken token, String action,
-                                                               ThreadContext context) {
-        return authenticationError("unable to authenticate user [{}] for action [{}]", token.principal(), action);
+            ThreadContext context) {
+        return createAuthenticationError("unable to authenticate user [{}] for action [{}]", null, token.principal(), action);
     }
 
     @Override
     public ElasticsearchSecurityException exceptionProcessingRequest(RestRequest request, Exception e, ThreadContext context) {
-        if (e instanceof ElasticsearchSecurityException) {
-            assert ((ElasticsearchSecurityException) e).status() == RestStatus.UNAUTHORIZED;
-            assert ((ElasticsearchSecurityException) e).getHeader("WWW-Authenticate").size() == 1;
-            return (ElasticsearchSecurityException) e;
-        }
-        return authenticationError("error attempting to authenticate request", e);
+        return createAuthenticationError("error attempting to authenticate request", e, (Object[]) null);
     }
 
     @Override
     public ElasticsearchSecurityException exceptionProcessingRequest(TransportMessage message, String action, Exception e,
-                                                                     ThreadContext context) {
-        if (e instanceof ElasticsearchSecurityException) {
-            assert ((ElasticsearchSecurityException) e).status() == RestStatus.UNAUTHORIZED;
-            assert ((ElasticsearchSecurityException) e).getHeader("WWW-Authenticate").size() == 1;
-            return (ElasticsearchSecurityException) e;
-        }
-        return authenticationError("error attempting to authenticate request", e);
+            ThreadContext context) {
+        return createAuthenticationError("error attempting to authenticate request", e, (Object[]) null);
     }
 
     @Override
     public ElasticsearchSecurityException missingToken(RestRequest request, ThreadContext context) {
-        return authenticationError("missing authentication token for REST request [{}]", request.uri());
+        return createAuthenticationError("missing authentication token for REST request [{}]", null, request.uri());
     }
 
     @Override
     public ElasticsearchSecurityException missingToken(TransportMessage message, String action, ThreadContext context) {
-        return authenticationError("missing authentication token for action [{}]", action);
+        return createAuthenticationError("missing authentication token for action [{}]", null, action);
     }
 
     @Override
     public ElasticsearchSecurityException authenticationRequired(String action, ThreadContext context) {
-        return authenticationError("action [{}] requires authentication", action);
+        return createAuthenticationError("action [{}] requires authentication", null, action);
+    }
+
+    /**
+     * Creates an instance of {@link ElasticsearchSecurityException} with
+     * {@link RestStatus#UNAUTHORIZED} status.
+     * <p>
+     * Also adds default failure response headers as configured for this
+     * {@link DefaultAuthenticationFailureHandler}
+     * <p>
+     * It may replace existing response headers if the cause is an instance of
+     * {@link ElasticsearchSecurityException}
+     *
+     * @param message error message
+     * @param t cause, if it is an instance of
+     *            {@link ElasticsearchSecurityException} asserts status is
+     *            RestStatus.UNAUTHORIZED and adds headers to it, else it will
+     *            create a new instance of {@link ElasticsearchSecurityException}
+     * @param args error message args
+     * @return instance of {@link ElasticsearchSecurityException}
+     */
+    private ElasticsearchSecurityException createAuthenticationError(final String message, final Throwable t, final Object... args) {
+        final ElasticsearchSecurityException ese;
+        final boolean containsNegotiateWithToken;
+        if (t instanceof ElasticsearchSecurityException) {
+            assert ((ElasticsearchSecurityException) t).status() == RestStatus.UNAUTHORIZED;
+            ese = (ElasticsearchSecurityException) t;
+            if (ese.getHeader("WWW-Authenticate") != null && ese.getHeader("WWW-Authenticate").isEmpty() == false) {
+                /**
+                 * If 'WWW-Authenticate' header is present with 'Negotiate ' then do not
+                 * replace. In case of kerberos spnego mechanism, we use
+                 * 'WWW-Authenticate' header value to communicate outToken to peer.
+                 */
+                containsNegotiateWithToken =
+                        ese.getHeader("WWW-Authenticate").stream()
+                                .anyMatch(s -> s != null && s.regionMatches(true, 0, "Negotiate ", 0, "Negotiate ".length()));
+            } else {
+                containsNegotiateWithToken = false;
+            }
+        } else {
+            ese = authenticationError(message, t, args);
+            containsNegotiateWithToken = false;
+        }
+        defaultFailureResponseHeaders.entrySet().stream().forEach((e) -> {
+            if (containsNegotiateWithToken && e.getKey().equalsIgnoreCase("WWW-Authenticate")) {
+                return;
+            }
+            // If it is already present then it will replace the existing header.
+            ese.addHeader(e.getKey(), e.getValue());
+        });
+        return ese;
     }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Realm.java

@@ -8,9 +8,12 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.security.user.User;
 
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -56,6 +59,18 @@ public int order() {
         return config.order;
     }
 
+    /**
+     * Each realm can define response headers to be sent on failure.
+     * <p>
+     * By default it adds 'WWW-Authenticate' header with auth scheme 'Basic'.
+     *
+     * @return Map of authentication failure response headers.
+     */
+    public Map<String, List<String>> getAuthenticationFailureHeaders() {
+        return Collections.singletonMap("WWW-Authenticate",
+                Collections.singletonList("Basic realm=\"" + XPackField.SECURITY + "\" charset=\"UTF-8\""));
+    }
+
     @Override
     public int compareTo(Realm other) {
         int result = Integer.compare(config.order, other.config.order);

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/kerberos/KerberosRealmSettings.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.authc.kerberos;
+
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.set.Sets;
+
+import java.util.Set;
+
+/**
+ * Kerberos Realm settings
+ */
+public final class KerberosRealmSettings {
+    public static final String TYPE = "kerberos";
+
+    /**
+     * Kerberos key tab for Elasticsearch service<br>
+     * Uses single key tab for multiple service accounts.
+     */
+    public static final Setting<String> HTTP_SERVICE_KEYTAB_PATH =
+            Setting.simpleString("keytab.path", Property.NodeScope);
+    public static final Setting<Boolean> SETTING_KRB_DEBUG_ENABLE =
+            Setting.boolSetting("krb.debug", Boolean.FALSE, Property.NodeScope);
+    public static final Setting<Boolean> SETTING_REMOVE_REALM_NAME =
+            Setting.boolSetting("remove_realm_name", Boolean.FALSE, Property.NodeScope);
+
+    // Cache
+    private static final TimeValue DEFAULT_TTL = TimeValue.timeValueMinutes(20);
+    private static final int DEFAULT_MAX_USERS = 100_000; // 100k users
+    public static final Setting<TimeValue> CACHE_TTL_SETTING = Setting.timeSetting("cache.ttl", DEFAULT_TTL, Setting.Property.NodeScope);
+    public static final Setting<Integer> CACHE_MAX_USERS_SETTING =
+            Setting.intSetting("cache.max_users", DEFAULT_MAX_USERS, Property.NodeScope);
+
+    private KerberosRealmSettings() {
+    }
+
+    /**
+     * @return the valid set of {@link Setting}s for a {@value #TYPE} realm
+     */
+    public static Set<Setting<?>> getSettings() {
+        return Sets.newHashSet(HTTP_SERVICE_KEYTAB_PATH, CACHE_TTL_SETTING, CACHE_MAX_USERS_SETTING, SETTING_KRB_DEBUG_ENABLE,
+                SETTING_REMOVE_REALM_NAME);
+    }
+}

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandlerTests.java

@@ -0,0 +1,114 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.authc;
+
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.XPackField;
+import org.mockito.Mockito;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.sameInstance;
+
+public class DefaultAuthenticationFailureHandlerTests extends ESTestCase {
+
+    public void testAuthenticationRequired() {
+        final boolean testDefault = randomBoolean();
+        final String basicAuthScheme = "Basic realm=\"" + XPackField.SECURITY + "\" charset=\"UTF-8\"";
+        final String bearerAuthScheme = "Bearer realm=\"" + XPackField.SECURITY + "\"";
+        final DefaultAuthenticationFailureHandler failuerHandler;
+        if (testDefault) {
+            failuerHandler = new DefaultAuthenticationFailureHandler();
+        } else {
+            final Map<String, List<String>> failureResponeHeaders = new HashMap<>();
+            failureResponeHeaders.put("WWW-Authenticate", Arrays.asList(basicAuthScheme, bearerAuthScheme));
+            failuerHandler = new DefaultAuthenticationFailureHandler(failureResponeHeaders);
+        }
+        assertThat(failuerHandler, is(notNullValue()));
+        final ElasticsearchSecurityException ese =
+                failuerHandler.authenticationRequired("someaction", new ThreadContext(Settings.builder().build()));
+        assertThat(ese, is(notNullValue()));
+        assertThat(ese.getMessage(), equalTo("action [someaction] requires authentication"));
+        assertThat(ese.getHeader("WWW-Authenticate"), is(notNullValue()));
+        if (testDefault) {
+            assertWWWAuthenticateWithSchemes(ese, basicAuthScheme);
+        } else {
+            assertWWWAuthenticateWithSchemes(ese, basicAuthScheme, bearerAuthScheme);
+        }
+    }
+
+    public void testExceptionProcessingRequest() {
+        final String basicAuthScheme = "Basic realm=\"" + XPackField.SECURITY + "\" charset=\"UTF-8\"";
+        final String bearerAuthScheme = "Bearer realm=\"" + XPackField.SECURITY + "\"";
+        final String negotiateAuthScheme = randomFrom("Negotiate", "Negotiate Ijoijksdk");
+        final Map<String, List<String>> failureResponeHeaders = new HashMap<>();
+        failureResponeHeaders.put("WWW-Authenticate", Arrays.asList(basicAuthScheme, bearerAuthScheme, negotiateAuthScheme));
+        final DefaultAuthenticationFailureHandler failuerHandler = new DefaultAuthenticationFailureHandler(failureResponeHeaders);
+
+        assertThat(failuerHandler, is(notNullValue()));
+        final boolean causeIsElasticsearchSecurityException = randomBoolean();
+        final boolean causeIsEseAndUnauthorized = causeIsElasticsearchSecurityException && randomBoolean();
+        final ElasticsearchSecurityException eseCause = (causeIsEseAndUnauthorized)
+                ? new ElasticsearchSecurityException("unauthorized", RestStatus.UNAUTHORIZED, null, (Object[]) null)
+                : new ElasticsearchSecurityException("different error", RestStatus.BAD_REQUEST, null, (Object[]) null);
+        final Exception cause = causeIsElasticsearchSecurityException ? eseCause : new Exception("other error");
+        final boolean withAuthenticateHeader = randomBoolean();
+        final String selectedScheme = randomFrom(bearerAuthScheme, basicAuthScheme, negotiateAuthScheme);
+        if (withAuthenticateHeader) {
+            eseCause.addHeader("WWW-Authenticate", Collections.singletonList(selectedScheme));
+        }
+
+        if (causeIsElasticsearchSecurityException) {
+            if (causeIsEseAndUnauthorized) {
+                final ElasticsearchSecurityException ese = failuerHandler.exceptionProcessingRequest(Mockito.mock(RestRequest.class), cause,
+                        new ThreadContext(Settings.builder().build()));
+                assertThat(ese, is(notNullValue()));
+                assertThat(ese.getHeader("WWW-Authenticate"), is(notNullValue()));
+                assertThat(ese, is(sameInstance(cause)));
+                if (withAuthenticateHeader == false) {
+                    assertWWWAuthenticateWithSchemes(ese, basicAuthScheme, bearerAuthScheme, negotiateAuthScheme);
+                } else {
+                    if (selectedScheme.contains("Negotiate ")) {
+                        assertWWWAuthenticateWithSchemes(ese, selectedScheme);
+                    } else {
+                        assertWWWAuthenticateWithSchemes(ese, basicAuthScheme, bearerAuthScheme, negotiateAuthScheme);
+                    }
+                }
+                assertThat(ese.getMessage(), equalTo("unauthorized"));
+            } else {
+                expectThrows(AssertionError.class, () -> failuerHandler.exceptionProcessingRequest(Mockito.mock(RestRequest.class), cause,
+                        new ThreadContext(Settings.builder().build())));
+            }
+        } else {
+            final ElasticsearchSecurityException ese = failuerHandler.exceptionProcessingRequest(Mockito.mock(RestRequest.class), cause,
+                    new ThreadContext(Settings.builder().build()));
+            assertThat(ese, is(notNullValue()));
+            assertThat(ese.getHeader("WWW-Authenticate"), is(notNullValue()));
+            assertThat(ese.getMessage(), equalTo("error attempting to authenticate request"));
+            assertWWWAuthenticateWithSchemes(ese, basicAuthScheme, bearerAuthScheme, negotiateAuthScheme);
+        }
+
+    }
+
+    private void assertWWWAuthenticateWithSchemes(final ElasticsearchSecurityException ese, final String... schemes) {
+        assertThat(ese.getHeader("WWW-Authenticate").size(), is(schemes.length));
+        assertThat(ese.getHeader("WWW-Authenticate"), contains(schemes));
+    }
+}