From 21b9170dec8601b5509142a0fb3fa5389614232d Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 15 May 2018 12:13:24 -0700 Subject: [PATCH 01/34] Security: Remove SecurityLifecycleService (#30526) This commit removes the SecurityLifecycleService, relegating its former functions of listening for cluster state updates to SecurityIndexManager and IndexAuditTrail. --- .../xpack/security/Security.java | 33 ++--- .../security/SecurityLifecycleService.java | 129 ------------------ .../security/audit/index/IndexAuditTrail.java | 35 ++++- .../security/authc/ExpiredTokenRemover.java | 4 +- .../xpack/security/authc/InternalRealms.java | 6 +- .../xpack/security/authc/TokenService.java | 49 ++++--- .../authc/esnative/NativeUsersStore.java | 38 +++--- .../authc/esnative/ReservedRealm.java | 12 +- .../mapper/NativeRoleMappingStore.java | 31 ++--- .../security/authz/AuthorizationService.java | 6 +- .../security/authz/AuthorizedIndices.java | 4 +- .../authz/store/CompositeRolesStore.java | 3 - .../authz/store/NativeRolesStore.java | 41 +++--- .../support/SecurityIndexManager.java | 23 +++- .../integration/ClearRolesCacheTests.java | 6 +- .../test/SecurityIntegTestCase.java | 2 +- .../xpack/security/SecurityTests.java | 2 +- ...sportSamlInvalidateSessionActionTests.java | 5 +- .../saml/TransportSamlLogoutActionTests.java | 5 +- .../user/TransportGetUsersActionTests.java | 13 +- .../user/TransportPutUserActionTests.java | 5 +- .../authc/AuthenticationServiceTests.java | 8 +- .../security/authc/InternalRealmsTests.java | 8 +- .../security/authc/TokenAuthIntegTests.java | 10 +- .../security/authc/TokenServiceTests.java | 34 ++--- .../esnative/ESNativeMigrateToolTests.java | 7 +- .../authc/esnative/NativeRealmIntegTests.java | 2 +- .../authc/esnative/NativeUsersStoreTests.java | 11 +- .../authc/esnative/ReservedRealmTests.java | 32 ++--- .../mapper/NativeRoleMappingStoreTests.java | 7 +- .../authz/AuthorizationServiceTests.java | 2 +- .../authz/AuthorizedIndicesTests.java | 8 +- .../authz/IndicesAndAliasesResolverTests.java | 14 +- .../authz/store/NativeRolesStoreTests.java | 13 +- .../support/SecurityIndexManagerTests.java | 6 +- .../security/test/SecurityTestUtils.java | 2 +- .../xpack/security/user/XPackUserTests.java | 6 +- 37 files changed, 247 insertions(+), 375 deletions(-) delete mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityLifecycleService.java diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index 4e4f86baec768..44f5afd4bdb07 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -7,6 +7,7 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.util.SetOnce; +import org.elasticsearch.ElasticsearchTimeoutException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; @@ -16,6 +17,7 @@ import org.elasticsearch.bootstrap.BootstrapCheck; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; @@ -232,7 +234,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_FORMAT_SETTING; import static org.elasticsearch.xpack.core.XPackSettings.HTTP_SSL_ENABLED; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_TEMPLATE_NAME; -import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_INDEX_FORMAT; public class Security extends Plugin implements ActionPlugin, IngestPlugin, NetworkPlugin, ClusterPlugin, @@ -261,6 +263,8 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw private final SetOnce threadContext = new SetOnce<>(); private final SetOnce tokenService = new SetOnce<>(); private final SetOnce securityActionFilter = new SetOnce<>(); + private final SetOnce securityIndex = new SetOnce<>(); + private final SetOnce indexAuditTrail = new SetOnce<>(); private final List bootstrapChecks; private final List securityExtensions = new ArrayList<>(); @@ -368,7 +372,6 @@ Collection createComponents(Client client, ThreadPool threadPool, Cluste components.add(securityContext.get()); // audit trails construction - IndexAuditTrail indexAuditTrail = null; Set auditTrails = new LinkedHashSet<>(); if (XPackSettings.AUDIT_ENABLED.get(settings)) { List outputs = AUDIT_OUTPUTS_SETTING.get(settings); @@ -383,8 +386,8 @@ Collection createComponents(Client client, ThreadPool threadPool, Cluste auditTrails.add(new LoggingAuditTrail(settings, clusterService, threadPool)); break; case IndexAuditTrail.NAME: - indexAuditTrail = new IndexAuditTrail(settings, client, threadPool, clusterService); - auditTrails.add(indexAuditTrail); + indexAuditTrail.set(new IndexAuditTrail(settings, client, threadPool, clusterService)); + auditTrails.add(indexAuditTrail.get()); break; default: throw new IllegalArgumentException("Unknown audit trail output [" + output + "]"); @@ -396,20 +399,20 @@ Collection createComponents(Client client, ThreadPool threadPool, Cluste components.add(auditTrailService); this.auditTrailService.set(auditTrailService); - final SecurityLifecycleService securityLifecycleService = - new SecurityLifecycleService(settings, clusterService, threadPool, client, indexAuditTrail); - final TokenService tokenService = new TokenService(settings, Clock.systemUTC(), client, securityLifecycleService, clusterService); + securityIndex.set(new SecurityIndexManager(settings, client, SecurityIndexManager.SECURITY_INDEX_NAME, clusterService)); + + final TokenService tokenService = new TokenService(settings, Clock.systemUTC(), client, securityIndex.get(), clusterService); this.tokenService.set(tokenService); components.add(tokenService); // realms construction - final NativeUsersStore nativeUsersStore = new NativeUsersStore(settings, client, securityLifecycleService); - final NativeRoleMappingStore nativeRoleMappingStore = new NativeRoleMappingStore(settings, client, securityLifecycleService); + final NativeUsersStore nativeUsersStore = new NativeUsersStore(settings, client, securityIndex.get()); + final NativeRoleMappingStore nativeRoleMappingStore = new NativeRoleMappingStore(settings, client, securityIndex.get()); final AnonymousUser anonymousUser = new AnonymousUser(settings); final ReservedRealm reservedRealm = new ReservedRealm(env, settings, nativeUsersStore, - anonymousUser, securityLifecycleService, threadPool.getThreadContext()); + anonymousUser, securityIndex.get(), threadPool.getThreadContext()); Map realmFactories = new HashMap<>(InternalRealms.getFactories(threadPool, resourceWatcherService, - getSslService(), nativeUsersStore, nativeRoleMappingStore, securityLifecycleService)); + getSslService(), nativeUsersStore, nativeRoleMappingStore, securityIndex.get())); for (SecurityExtension extension : securityExtensions) { Map newRealms = extension.getRealms(resourceWatcherService); for (Map.Entry entry : newRealms.entrySet()) { @@ -424,7 +427,7 @@ Collection createComponents(Client client, ThreadPool threadPool, Cluste components.add(realms); components.add(reservedRealm); - securityLifecycleService.securityIndex().addIndexStateListener(nativeRoleMappingStore::onSecurityIndexStateChange); + securityIndex.get().addIndexStateListener(nativeRoleMappingStore::onSecurityIndexStateChange); AuthenticationFailureHandler failureHandler = null; String extensionName = null; @@ -449,7 +452,7 @@ Collection createComponents(Client client, ThreadPool threadPool, Cluste components.add(authcService.get()); final FileRolesStore fileRolesStore = new FileRolesStore(settings, env, resourceWatcherService, getLicenseState()); - final NativeRolesStore nativeRolesStore = new NativeRolesStore(settings, client, getLicenseState(), securityLifecycleService); + final NativeRolesStore nativeRolesStore = new NativeRolesStore(settings, client, getLicenseState(), securityIndex.get()); final ReservedRolesStore reservedRolesStore = new ReservedRolesStore(); List, ActionListener>>> rolesProviders = new ArrayList<>(); for (SecurityExtension extension : securityExtensions) { @@ -457,7 +460,7 @@ Collection createComponents(Client client, ThreadPool threadPool, Cluste } final CompositeRolesStore allRolesStore = new CompositeRolesStore(settings, fileRolesStore, nativeRolesStore, reservedRolesStore, rolesProviders, threadPool.getThreadContext(), getLicenseState()); - securityLifecycleService.securityIndex().addIndexStateListener(allRolesStore::onSecurityIndexStateChange); + securityIndex.get().addIndexStateListener(allRolesStore::onSecurityIndexStateChange); // to keep things simple, just invalidate all cached entries on license change. this happens so rarely that the impact should be // minimal getLicenseState().addListener(allRolesStore::invalidateAll); @@ -468,8 +471,6 @@ Collection createComponents(Client client, ThreadPool threadPool, Cluste components.add(allRolesStore); // for SecurityFeatureSet and clear roles cache components.add(authzService); - components.add(securityLifecycleService); - ipFilter.set(new IPFilter(settings, auditTrailService, clusterService.getClusterSettings(), getLicenseState())); components.add(ipFilter.get()); DestructiveOperations destructiveOperations = new DestructiveOperations(settings, clusterService.getClusterSettings()); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityLifecycleService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityLifecycleService.java deleted file mode 100644 index 9d05ec3b71d6b..0000000000000 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityLifecycleService.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.security; - -import org.apache.logging.log4j.Logger; -import org.elasticsearch.Version; -import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.ClusterChangedEvent; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ClusterStateListener; -import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.cluster.health.ClusterIndexHealth; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.component.LifecycleListener; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.AbstractRunnable; -import org.elasticsearch.gateway.GatewayService; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.security.audit.index.IndexAuditTrail; -import org.elasticsearch.xpack.security.support.SecurityIndexManager; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.function.BiConsumer; -import java.util.function.Consumer; -import java.util.function.Predicate; - -/** - * This class is used to provide a lifecycle for services that is based on the cluster's state - * rather than the typical lifecycle that is used to start services as part of the node startup. - * - * This type of lifecycle is necessary for services that need to perform actions that require the - * cluster to be in a certain state; some examples are storing index templates and creating indices. - * These actions would most likely fail from within a plugin if executed in the - * {@link org.elasticsearch.common.component.AbstractLifecycleComponent#doStart()} method. - * However, if the startup of these services waits for the cluster to form and recover indices then - * it will be successful. This lifecycle service allows for this to happen by listening for - * {@link ClusterChangedEvent} and checking if the services can start. Additionally, the service - * also provides hooks for stop and close functionality. - */ -public class SecurityLifecycleService extends AbstractComponent implements ClusterStateListener { - - public static final String INTERNAL_SECURITY_INDEX = SecurityIndexManager.INTERNAL_SECURITY_INDEX; - public static final String SECURITY_INDEX_NAME = ".security"; - - private final Settings settings; - private final ThreadPool threadPool; - private final IndexAuditTrail indexAuditTrail; - - private final SecurityIndexManager securityIndex; - - public SecurityLifecycleService(Settings settings, ClusterService clusterService, - ThreadPool threadPool, Client client, - @Nullable IndexAuditTrail indexAuditTrail) { - super(settings); - this.settings = settings; - this.threadPool = threadPool; - this.indexAuditTrail = indexAuditTrail; - this.securityIndex = new SecurityIndexManager(settings, client, SECURITY_INDEX_NAME); - clusterService.addListener(this); - clusterService.addLifecycleListener(new LifecycleListener() { - @Override - public void beforeStop() { - close(); - } - }); - } - - @Override - public void clusterChanged(ClusterChangedEvent event) { - final ClusterState state = event.state(); - if (state.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) { - // wait until the gateway has recovered from disk, otherwise we think we don't have the - // .security index but they may not have been restored from the cluster state on disk - logger.debug("lifecycle service waiting until state has been recovered"); - return; - } - - securityIndex.clusterChanged(event); - - try { - if (Security.indexAuditLoggingEnabled(settings) && - indexAuditTrail.state() == IndexAuditTrail.State.INITIALIZED) { - if (indexAuditTrail.canStart(event)) { - threadPool.generic().execute(new AbstractRunnable() { - - @Override - public void onFailure(Exception throwable) { - logger.error("failed to start index audit trail services", throwable); - assert false : "security lifecycle services startup failed"; - } - - @Override - public void doRun() { - indexAuditTrail.start(); - } - }); - } - } - } catch (Exception e) { - logger.error("failed to start index audit trail", e); - } - } - - public SecurityIndexManager securityIndex() { - return securityIndex; - } - - // this is called in a lifecycle listener beforeStop on the cluster service - private void close() { - if (indexAuditTrail != null) { - try { - indexAuditTrail.stop(); - } catch (Exception e) { - logger.error("failed to stop audit trail module", e); - } - } - } - - public static List indexNames() { - return Collections.unmodifiableList(Arrays.asList(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)); - } -} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java index 590c2bc5ecd4e..db7475a89727f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java @@ -20,6 +20,7 @@ import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.metadata.AliasOrIndex; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; @@ -29,12 +30,14 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.component.LifecycleListener; import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -110,7 +113,7 @@ /** * Audit trail implementation that writes events into an index. */ -public class IndexAuditTrail extends AbstractComponent implements AuditTrail { +public class IndexAuditTrail extends AbstractComponent implements AuditTrail, ClusterStateListener { public static final String NAME = "index"; public static final String DOC_TYPE = "doc"; @@ -199,6 +202,13 @@ public IndexAuditTrail(Settings settings, Client client, ThreadPool threadPool, } else { this.client = initializeRemoteClient(settings, logger); } + clusterService.addListener(this); + clusterService.addLifecycleListener(new LifecycleListener() { + @Override + public void beforeStop() { + stop(); + } + }); } @@ -206,6 +216,29 @@ public State state() { return state.get(); } + @Override + public void clusterChanged(ClusterChangedEvent event) { + try { + if (state() == IndexAuditTrail.State.INITIALIZED && canStart(event)) { + threadPool.generic().execute(new AbstractRunnable() { + + @Override + public void onFailure(Exception throwable) { + logger.error("failed to start index audit trail services", throwable); + assert false : "security lifecycle services startup failed"; + } + + @Override + public void doRun() { + start(); + } + }); + } + } catch (Exception e) { + logger.error("failed to start index audit trail", e); + } + } + /** * This method determines if this service can be started based on the state in the {@link ClusterChangedEvent} and * if the node is the master or not. When using remote indexing, a call to the remote cluster will be made to retrieve diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java index a46d6131c6035..b8ae5c944419a 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java @@ -22,7 +22,7 @@ import org.elasticsearch.index.reindex.ScrollableHitSource; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool.Names; -import org.elasticsearch.xpack.security.SecurityLifecycleService; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import java.time.Instant; import java.time.temporal.ChronoUnit; @@ -50,7 +50,7 @@ final class ExpiredTokenRemover extends AbstractRunnable { @Override public void doRun() { - SearchRequest searchRequest = new SearchRequest(SecurityLifecycleService.SECURITY_INDEX_NAME); + SearchRequest searchRequest = new SearchRequest(SecurityIndexManager.SECURITY_INDEX_NAME); DeleteByQueryRequest expiredDbq = new DeleteByQueryRequest(searchRequest); if (timeout != TimeValue.MINUS_ONE) { expiredDbq.setTimeout(timeout); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/InternalRealms.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/InternalRealms.java index b50264a73e949..1e38e6fd10391 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/InternalRealms.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/InternalRealms.java @@ -20,7 +20,6 @@ import org.elasticsearch.xpack.core.security.authc.pki.PkiRealmSettings; import org.elasticsearch.xpack.core.security.authc.saml.SamlRealmSettings; import org.elasticsearch.xpack.core.ssl.SSLService; -import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.authc.esnative.NativeRealm; import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore; import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm; @@ -30,6 +29,7 @@ import org.elasticsearch.xpack.security.authc.saml.SamlRealm; import org.elasticsearch.xpack.security.authc.support.RoleMappingFileBootstrapCheck; import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import java.util.ArrayList; import java.util.Arrays; @@ -90,13 +90,13 @@ static boolean isStandardRealm(String type) { public static Map getFactories(ThreadPool threadPool, ResourceWatcherService resourceWatcherService, SSLService sslService, NativeUsersStore nativeUsersStore, NativeRoleMappingStore nativeRoleMappingStore, - SecurityLifecycleService securityLifecycleService) { + SecurityIndexManager securityIndex) { Map map = new HashMap<>(); map.put(FileRealmSettings.TYPE, config -> new FileRealm(config, resourceWatcherService)); map.put(NativeRealmSettings.TYPE, config -> { final NativeRealm nativeRealm = new NativeRealm(config, nativeUsersStore); - securityLifecycleService.securityIndex().addIndexStateListener(nativeRealm::onSecurityIndexStateChange); + securityIndex.addIndexStateListener(nativeRealm::onSecurityIndexStateChange); return nativeRealm; }); map.put(LdapRealmSettings.AD_TYPE, config -> new LdapRealm(LdapRealmSettings.AD_TYPE, config, sslService, diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index 28098faa50ea6..d23415f87dfcc 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -9,7 +9,6 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.elasticsearch.core.internal.io.IOUtils; -import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.UnicodeUtil; import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.ExceptionsHelper; @@ -70,7 +69,7 @@ import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.KeyAndTimestamp; import org.elasticsearch.xpack.core.security.authc.TokenMetaData; -import org.elasticsearch.xpack.security.SecurityLifecycleService; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import javax.crypto.Cipher; import javax.crypto.CipherInputStream; @@ -161,7 +160,7 @@ public final class TokenService extends AbstractComponent { private final TimeValue expirationDelay; private final TimeValue deleteInterval; private final Client client; - private final SecurityLifecycleService lifecycleService; + private final SecurityIndexManager securityIndex; private final ExpiredTokenRemover expiredTokenRemover; private final boolean enabled; private volatile TokenKeys keyCache; @@ -176,7 +175,7 @@ public final class TokenService extends AbstractComponent { * @param client the client to use when checking for revocations */ public TokenService(Settings settings, Clock clock, Client client, - SecurityLifecycleService lifecycleService, ClusterService clusterService) throws GeneralSecurityException { + SecurityIndexManager securityIndex, ClusterService clusterService) throws GeneralSecurityException { super(settings); byte[] saltArr = new byte[SALT_BYTES]; secureRandom.nextBytes(saltArr); @@ -185,7 +184,7 @@ public TokenService(Settings settings, Clock clock, Client client, this.clock = clock.withZone(ZoneOffset.UTC); this.expirationDelay = TOKEN_EXPIRATION.get(settings); this.client = client; - this.lifecycleService = lifecycleService; + this.securityIndex = securityIndex; this.lastExpirationRunMs = client.threadPool().relativeTimeInMillis(); this.deleteInterval = DELETE_INTERVAL.get(settings); this.enabled = isTokenServiceEnabled(settings); @@ -245,12 +244,12 @@ public void createUserToken(Authentication authentication, Authentication origin .endObject(); builder.endObject(); IndexRequest request = - client.prepareIndex(SecurityLifecycleService.SECURITY_INDEX_NAME, TYPE, getTokenDocumentId(userToken)) + client.prepareIndex(SecurityIndexManager.SECURITY_INDEX_NAME, TYPE, getTokenDocumentId(userToken)) .setOpType(OpType.CREATE) .setSource(builder) .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) .request(); - lifecycleService.securityIndex().prepareIndexIfNeededThenExecute(listener::onFailure, () -> + securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> executeAsyncWithOrigin(client, SECURITY_ORIGIN, IndexAction.INSTANCE, request, ActionListener.wrap(indexResponse -> listener.onResponse(new Tuple<>(userToken, refreshToken)), listener::onFailure)) @@ -354,9 +353,9 @@ void decodeToken(String token, ActionListener listener) throws IOExce if (version.onOrAfter(Version.V_6_2_0)) { // we only have the id and need to get the token from the doc! decryptTokenId(in, cipher, version, ActionListener.wrap(tokenId -> - lifecycleService.securityIndex().prepareIndexIfNeededThenExecute(listener::onFailure, () -> { + securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> { final GetRequest getRequest = - client.prepareGet(SecurityLifecycleService.SECURITY_INDEX_NAME, TYPE, + client.prepareGet(SecurityIndexManager.SECURITY_INDEX_NAME, TYPE, getTokenDocumentId(tokenId)).request(); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, getRequest, ActionListener.wrap(response -> { @@ -517,14 +516,14 @@ private void indexBwcInvalidation(UserToken userToken, ActionListener l listener.onFailure(invalidGrantException("failed to invalidate token")); } else { final String invalidatedTokenId = getInvalidatedTokenDocumentId(userToken); - IndexRequest indexRequest = client.prepareIndex(SecurityLifecycleService.SECURITY_INDEX_NAME, TYPE, invalidatedTokenId) + IndexRequest indexRequest = client.prepareIndex(SecurityIndexManager.SECURITY_INDEX_NAME, TYPE, invalidatedTokenId) .setOpType(OpType.CREATE) .setSource("doc_type", INVALIDATED_TOKEN_DOC_TYPE, "expiration_time", expirationEpochMilli) .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) .request(); final String tokenDocId = getTokenDocumentId(userToken); final Version version = userToken.getVersion(); - lifecycleService.securityIndex().prepareIndexIfNeededThenExecute(listener::onFailure, () -> + securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, indexRequest, ActionListener.wrap(indexResponse -> { ActionListener wrappedListener = @@ -561,12 +560,12 @@ private void indexInvalidation(String tokenDocId, Version version, ActionListene if (attemptCount.get() > 5) { listener.onFailure(invalidGrantException("failed to invalidate token")); } else { - UpdateRequest request = client.prepareUpdate(SecurityLifecycleService.SECURITY_INDEX_NAME, TYPE, tokenDocId) + UpdateRequest request = client.prepareUpdate(SecurityIndexManager.SECURITY_INDEX_NAME, TYPE, tokenDocId) .setDoc(srcPrefix, Collections.singletonMap("invalidated", true)) .setVersion(documentVersion) .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) .request(); - lifecycleService.securityIndex().prepareIndexIfNeededThenExecute(listener::onFailure, () -> + securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, request, ActionListener.wrap(updateResponse -> { if (updateResponse.getGetResult() != null @@ -593,7 +592,7 @@ private void indexInvalidation(String tokenDocId, Version version, ActionListene || isShardNotAvailableException(cause)) { attemptCount.incrementAndGet(); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareGet(SecurityLifecycleService.SECURITY_INDEX_NAME, TYPE, tokenDocId).request(), + client.prepareGet(SecurityIndexManager.SECURITY_INDEX_NAME, TYPE, tokenDocId).request(), ActionListener.wrap(getResult -> { if (getResult.isExists()) { Map source = getResult.getSource(); @@ -658,14 +657,14 @@ private void findTokenFromRefreshToken(String refreshToken, ActionListener 5) { listener.onFailure(invalidGrantException("could not refresh the requested token")); } else { - SearchRequest request = client.prepareSearch(SecurityLifecycleService.SECURITY_INDEX_NAME) + SearchRequest request = client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) .setQuery(QueryBuilders.boolQuery() .filter(QueryBuilders.termQuery("doc_type", "token")) .filter(QueryBuilders.termQuery("refresh_token.token", refreshToken))) .setVersion(true) .request(); - lifecycleService.securityIndex().prepareIndexIfNeededThenExecute(listener::onFailure, () -> + securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, request, ActionListener.wrap(searchResponse -> { if (searchResponse.isTimedOut()) { @@ -702,7 +701,7 @@ private void innerRefresh(String tokenDocId, Authentication userAuth, ActionList if (attemptCount.getAndIncrement() > 5) { listener.onFailure(invalidGrantException("could not refresh the requested token")); } else { - GetRequest getRequest = client.prepareGet(SecurityLifecycleService.SECURITY_INDEX_NAME, TYPE, tokenDocId).request(); + GetRequest getRequest = client.prepareGet(SecurityIndexManager.SECURITY_INDEX_NAME, TYPE, tokenDocId).request(); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, getRequest, ActionListener.wrap(response -> { if (response.isExists()) { @@ -723,7 +722,7 @@ private void innerRefresh(String tokenDocId, Authentication userAuth, ActionList in.setVersion(authVersion); Authentication authentication = new Authentication(in); UpdateRequest updateRequest = - client.prepareUpdate(SecurityLifecycleService.SECURITY_INDEX_NAME, TYPE, tokenDocId) + client.prepareUpdate(SecurityIndexManager.SECURITY_INDEX_NAME, TYPE, tokenDocId) .setVersion(response.getVersion()) .setDoc("refresh_token", Collections.singletonMap("refreshed", true)) .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) @@ -838,7 +837,7 @@ public void findActiveTokensForRealm(String realmName, ActionListener supplier = client.threadPool().getThreadContext().newRestorableContext(false); - lifecycleService.securityIndex().prepareIndexIfNeededThenExecute(listener::onFailure, () -> + securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> ScrollHelper.fetchAllByEntity(client, request, new ContextPreservingActionListener<>(supplier, listener), this::parseHit)); } @@ -914,14 +913,14 @@ private void ensureEnabled() { * have been explicitly cleared. */ private void checkIfTokenIsRevoked(UserToken userToken, ActionListener listener) { - if (lifecycleService.securityIndex().indexExists() == false) { + if (securityIndex.indexExists() == false) { // index doesn't exist so the token is considered valid. listener.onResponse(userToken); } else { - lifecycleService.securityIndex().prepareIndexIfNeededThenExecute(listener::onFailure, () -> { + securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> { MultiGetRequest mGetRequest = client.prepareMultiGet() - .add(SecurityLifecycleService.SECURITY_INDEX_NAME, TYPE, getInvalidatedTokenDocumentId(userToken)) - .add(SecurityLifecycleService.SECURITY_INDEX_NAME, TYPE, getTokenDocumentId(userToken)) + .add(SecurityIndexManager.SECURITY_INDEX_NAME, TYPE, getInvalidatedTokenDocumentId(userToken)) + .add(SecurityIndexManager.SECURITY_INDEX_NAME, TYPE, getTokenDocumentId(userToken)) .request(); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, mGetRequest, @@ -989,7 +988,7 @@ private Instant getExpirationTime(Instant now) { } private void maybeStartTokenRemover() { - if (lifecycleService.securityIndex().isAvailable()) { + if (securityIndex.isAvailable()) { if (client.threadPool().relativeTimeInMillis() - lastExpirationRunMs > deleteInterval.getMillis()) { expiredTokenRemover.submit(client.threadPool()); lastExpirationRunMs = client.threadPool().relativeTimeInMillis(); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java index 381053d9633d7..1477c6dc88045 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java @@ -50,7 +50,7 @@ import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.core.security.user.User.Fields; import org.elasticsearch.xpack.core.security.user.XPackUser; -import org.elasticsearch.xpack.security.SecurityLifecycleService; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import java.util.Arrays; import java.util.Collection; @@ -64,7 +64,7 @@ import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; import static org.elasticsearch.xpack.core.ClientHelper.stashWithOrigin; -import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; /** * NativeUsersStore is a store for users that reads from an Elasticsearch index. This store is responsible for fetching the full @@ -83,12 +83,12 @@ public class NativeUsersStore extends AbstractComponent { private final Hasher hasher = Hasher.BCRYPT; private final Client client; - private volatile SecurityLifecycleService securityLifecycleService; + private final SecurityIndexManager securityIndex; - public NativeUsersStore(Settings settings, Client client, SecurityLifecycleService securityLifecycleService) { + public NativeUsersStore(Settings settings, Client client, SecurityIndexManager securityIndex) { super(settings); this.client = client; - this.securityLifecycleService = securityLifecycleService; + this.securityIndex = securityIndex; } /** @@ -114,7 +114,7 @@ public void getUsers(String[] userNames, final ActionListener> } }; - if (securityLifecycleService.securityIndex().indexExists() == false) { + if (securityIndex.indexExists() == false) { // TODO remove this short circuiting and fix tests that fail without this! listener.onResponse(Collections.emptyList()); } else if (userNames.length == 1) { // optimization for single user lookup @@ -123,7 +123,7 @@ public void getUsers(String[] userNames, final ActionListener> (uap) -> listener.onResponse(uap == null ? Collections.emptyList() : Collections.singletonList(uap.user())), handleException)); } else { - securityLifecycleService.securityIndex().prepareIndexIfNeededThenExecute(listener::onFailure, () -> { + securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> { final QueryBuilder query; if (userNames == null || userNames.length == 0) { query = QueryBuilders.termQuery(Fields.TYPE.getPreferredName(), USER_DOC_TYPE); @@ -154,11 +154,11 @@ public void getUsers(String[] userNames, final ActionListener> * Async method to retrieve a user and their password */ private void getUserAndPassword(final String user, final ActionListener listener) { - if (securityLifecycleService.securityIndex().indexExists() == false) { + if (securityIndex.indexExists() == false) { // TODO remove this short circuiting and fix tests that fail without this! listener.onResponse(null); } else { - securityLifecycleService.securityIndex().prepareIndexIfNeededThenExecute(listener::onFailure, () -> + securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, client.prepareGet(SECURITY_INDEX_NAME, INDEX_TYPE, getIdForUser(USER_DOC_TYPE, user)).request(), @@ -199,7 +199,7 @@ public void changePassword(final ChangePasswordRequest request, final ActionList docType = USER_DOC_TYPE; } - securityLifecycleService.securityIndex().prepareIndexIfNeededThenExecute(listener::onFailure, () -> { + securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> { executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, client.prepareUpdate(SECURITY_INDEX_NAME, INDEX_TYPE, getIdForUser(docType, username)) .setDoc(Requests.INDEX_CONTENT_TYPE, Fields.PASSWORD.getPreferredName(), @@ -237,7 +237,7 @@ public void onFailure(Exception e) { * has been indexed */ private void createReservedUser(String username, char[] passwordHash, RefreshPolicy refresh, ActionListener listener) { - securityLifecycleService.securityIndex().prepareIndexIfNeededThenExecute(listener::onFailure, () -> { + securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> { executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, client.prepareIndex(SECURITY_INDEX_NAME, INDEX_TYPE, getIdForUser(RESERVED_USER_TYPE, username)) @@ -279,7 +279,7 @@ public void putUser(final PutUserRequest request, final ActionListener private void updateUserWithoutPassword(final PutUserRequest putUserRequest, final ActionListener listener) { assert putUserRequest.passwordHash() == null; // We must have an existing document - securityLifecycleService.securityIndex().prepareIndexIfNeededThenExecute(listener::onFailure, () -> { + securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> { executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, client.prepareUpdate(SECURITY_INDEX_NAME, INDEX_TYPE, getIdForUser(USER_DOC_TYPE, putUserRequest.username())) @@ -322,7 +322,7 @@ public void onFailure(Exception e) { private void indexUser(final PutUserRequest putUserRequest, final ActionListener listener) { assert putUserRequest.passwordHash() != null; - securityLifecycleService.securityIndex().prepareIndexIfNeededThenExecute(listener::onFailure, () -> { + securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> { executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, client.prepareIndex(SECURITY_INDEX_NAME, INDEX_TYPE, getIdForUser(USER_DOC_TYPE, putUserRequest.username())) @@ -366,7 +366,7 @@ public void setEnabled(final String username, final boolean enabled, final Refre private void setRegularUserEnabled(final String username, final boolean enabled, final RefreshPolicy refreshPolicy, final ActionListener listener) { - securityLifecycleService.securityIndex().prepareIndexIfNeededThenExecute(listener::onFailure, () -> { + securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> { executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, client.prepareUpdate(SECURITY_INDEX_NAME, INDEX_TYPE, getIdForUser(USER_DOC_TYPE, username)) @@ -401,7 +401,7 @@ public void onFailure(Exception e) { private void setReservedUserEnabled(final String username, final boolean enabled, final RefreshPolicy refreshPolicy, boolean clearCache, final ActionListener listener) { - securityLifecycleService.securityIndex().prepareIndexIfNeededThenExecute(listener::onFailure, () -> { + securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> { executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, client.prepareUpdate(SECURITY_INDEX_NAME, INDEX_TYPE, getIdForUser(RESERVED_USER_TYPE, username)) @@ -431,7 +431,7 @@ public void onFailure(Exception e) { } public void deleteUser(final DeleteUserRequest deleteUserRequest, final ActionListener listener) { - securityLifecycleService.securityIndex().prepareIndexIfNeededThenExecute(listener::onFailure, () -> { + securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> { DeleteRequest request = client.prepareDelete(SECURITY_INDEX_NAME, INDEX_TYPE, getIdForUser(USER_DOC_TYPE, deleteUserRequest.username())).request(); request.setRefreshPolicy(deleteUserRequest.getRefreshPolicy()); @@ -470,11 +470,11 @@ void verifyPassword(String username, final SecureString password, ActionListener } void getReservedUserInfo(String username, ActionListener listener) { - if (securityLifecycleService.securityIndex().indexExists() == false) { + if (securityIndex.indexExists() == false) { // TODO remove this short circuiting and fix tests that fail without this! listener.onResponse(null); } else { - securityLifecycleService.securityIndex().prepareIndexIfNeededThenExecute(listener::onFailure, () -> + securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, client.prepareGet(SECURITY_INDEX_NAME, INDEX_TYPE, getIdForUser(RESERVED_USER_TYPE, username)).request(), @@ -514,7 +514,7 @@ public void onFailure(Exception e) { } void getAllReservedUserInfo(ActionListener> listener) { - securityLifecycleService.securityIndex().prepareIndexIfNeededThenExecute(listener::onFailure, () -> + securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, client.prepareSearch(SECURITY_INDEX_NAME) .setQuery(QueryBuilders.termQuery(Fields.TYPE.getPreferredName(), RESERVED_USER_TYPE)) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java index 199a1c1968408..7dbcea908722c 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java @@ -30,9 +30,9 @@ import org.elasticsearch.xpack.core.security.user.KibanaUser; import org.elasticsearch.xpack.core.security.user.LogstashSystemUser; import org.elasticsearch.xpack.core.security.user.User; -import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore.ReservedUserInfo; import org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import java.util.ArrayList; import java.util.Arrays; @@ -63,16 +63,16 @@ public class ReservedRealm extends CachingUsernamePasswordRealm { private final AnonymousUser anonymousUser; private final boolean realmEnabled; private final boolean anonymousEnabled; - private final SecurityLifecycleService securityLifecycleService; + private final SecurityIndexManager securityIndex; public ReservedRealm(Environment env, Settings settings, NativeUsersStore nativeUsersStore, AnonymousUser anonymousUser, - SecurityLifecycleService securityLifecycleService, ThreadContext threadContext) { + SecurityIndexManager securityIndex, ThreadContext threadContext) { super(TYPE, new RealmConfig(TYPE, Settings.EMPTY, settings, env, threadContext)); this.nativeUsersStore = nativeUsersStore; this.realmEnabled = XPackSettings.RESERVED_REALM_ENABLED_SETTING.get(settings); this.anonymousUser = anonymousUser; this.anonymousEnabled = AnonymousUser.isAnonymousEnabled(settings); - this.securityLifecycleService = securityLifecycleService; + this.securityIndex = securityIndex; final char[] hash = BOOTSTRAP_ELASTIC_PASSWORD.get(settings).length() == 0 ? EMPTY_PASSWORD_HASH : Hasher.BCRYPT.hash(BOOTSTRAP_ELASTIC_PASSWORD.get(settings)); bootstrapUserInfo = new ReservedUserInfo(hash, true, hash == EMPTY_PASSWORD_HASH); @@ -191,7 +191,7 @@ private void getUserInfo(final String username, ActionListener if (userIsDefinedForCurrentSecurityMapping(username) == false) { logger.debug("Marking user [{}] as disabled because the security mapping is not at the required version", username); listener.onResponse(DISABLED_DEFAULT_USER_INFO.deepClone()); - } else if (securityLifecycleService.securityIndex().indexExists() == false) { + } else if (securityIndex.indexExists() == false) { listener.onResponse(getDefaultUserInfo(username)); } else { nativeUsersStore.getReservedUserInfo(username, ActionListener.wrap((userInfo) -> { @@ -218,7 +218,7 @@ private ReservedUserInfo getDefaultUserInfo(String username) { private boolean userIsDefinedForCurrentSecurityMapping(String username) { final Version requiredVersion = getDefinedVersion(username); - return securityLifecycleService.securityIndex().checkMappingVersion(requiredVersion::onOrBefore); + return securityIndex.checkMappingVersion(requiredVersion::onOrBefore); } private Version getDefinedVersion(String username) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java index b6df03471242e..7df4114863de2 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java @@ -12,8 +12,6 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.cluster.health.ClusterIndexHealth; import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.component.AbstractComponent; @@ -36,7 +34,6 @@ import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping; import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.ExpressionModel; import org.elasticsearch.xpack.core.security.client.SecurityClient; -import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm; import org.elasticsearch.xpack.security.authc.support.UserRoleMapper; import org.elasticsearch.xpack.security.support.SecurityIndexManager; @@ -62,13 +59,13 @@ import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; import static org.elasticsearch.xpack.core.ClientHelper.stashWithOrigin; -import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isIndexDeleted; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isMoveFromRedToNonRed; /** * This store reads + writes {@link ExpressionRoleMapping role mappings} in an Elasticsearch - * {@link SecurityLifecycleService#SECURITY_INDEX_NAME index}. + * {@link SecurityIndexManager#SECURITY_INDEX_NAME index}. *
* The store is responsible for all read and write operations as well as * {@link #resolveRoles(UserData, ActionListener) resolving roles}. @@ -99,13 +96,13 @@ public void onFailure(Exception e) { }; private final Client client; - private final SecurityLifecycleService securityLifecycleService; + private final SecurityIndexManager securityIndex; private final List realmsToRefresh = new CopyOnWriteArrayList<>(); - public NativeRoleMappingStore(Settings settings, Client client, SecurityLifecycleService securityLifecycleService) { + public NativeRoleMappingStore(Settings settings, Client client, SecurityIndexManager securityIndex) { super(settings); this.client = client; - this.securityLifecycleService = securityLifecycleService; + this.securityIndex = securityIndex; } private String getNameFromId(String id) { @@ -122,7 +119,7 @@ private String getIdForName(String name) { * package private for unit testing */ void loadMappings(ActionListener> listener) { - if (securityLifecycleService.securityIndex().isIndexUpToDate() == false) { + if (securityIndex.isIndexUpToDate() == false) { listener.onFailure(new IllegalStateException( "Security index is not on the current version - the native realm will not be operational until " + "the upgrade API is run on the security index")); @@ -178,7 +175,7 @@ public void deleteRoleMapping(DeleteRoleMappingRequest request, ActionListener void modifyMapping(String name, CheckedBiConsumer, Exception> inner, Request request, ActionListener listener) { - if (securityLifecycleService.securityIndex().isIndexUpToDate() == false) { + if (securityIndex.isIndexUpToDate() == false) { listener.onFailure(new IllegalStateException( "Security index is not on the current version - the native realm will not be operational until " + "the upgrade API is run on the security index")); @@ -194,7 +191,7 @@ private void modifyMapping(String name, CheckedBiConsumer listener) { final ExpressionRoleMapping mapping = request.getMapping(); - securityLifecycleService.securityIndex().prepareIndexIfNeededThenExecute(listener::onFailure, () -> { + securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> { final XContentBuilder xContentBuilder; try { xContentBuilder = mapping.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS, true); @@ -224,7 +221,7 @@ public void onFailure(Exception e) { } private void innerDeleteMapping(DeleteRoleMappingRequest request, ActionListener listener) throws IOException { - if (securityLifecycleService.securityIndex().isIndexUpToDate() == false) { + if (securityIndex.isIndexUpToDate() == false) { listener.onFailure(new IllegalStateException( "Security index is not on the current version - the native realm will not be operational until " + "the upgrade API is run on the security index")); @@ -278,16 +275,16 @@ public void onFailure(Exception e) { } private void getMappings(ActionListener> listener) { - if (securityLifecycleService.securityIndex().isAvailable()) { + if (securityIndex.isAvailable()) { loadMappings(listener); } else { logger.info("The security index is not yet available - no role mappings can be loaded"); if (logger.isDebugEnabled()) { logger.debug("Security Index [{}] [exists: {}] [available: {}] [mapping up to date: {}]", SECURITY_INDEX_NAME, - securityLifecycleService.securityIndex().indexExists(), - securityLifecycleService.securityIndex().isAvailable(), - securityLifecycleService.securityIndex().isMappingUpToDate() + securityIndex.indexExists(), + securityIndex.isAvailable(), + securityIndex.isMappingUpToDate() ); } listener.onResponse(Collections.emptyList()); @@ -304,7 +301,7 @@ private void getMappings(ActionListener> listener) { * */ public void usageStats(ActionListener> listener) { - if (securityLifecycleService.securityIndex().indexExists() == false) { + if (securityIndex.indexExists() == false) { reportStats(listener, Collections.emptyList()); } else { getMappings(ActionListener.wrap(mappings -> reportStats(listener, mappings), listener::onFailure)); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java index 8ab48a0320602..19760ccab0202 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java @@ -60,11 +60,11 @@ import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.core.security.user.XPackSecurityUser; import org.elasticsearch.xpack.core.security.user.XPackUser; -import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.audit.AuditTrailService; import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm; import org.elasticsearch.xpack.security.authz.IndicesAndAliasesResolver.ResolvedIndices; import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import java.util.Arrays; import java.util.Collections; @@ -301,7 +301,7 @@ && isSuperuser(authentication.getUser()) == false) { // only the XPackUser is allowed to work with this index, but we should allow indices monitoring actions through for debugging // purposes. These monitor requests also sometimes resolve indices concretely and then requests them logger.debug("user [{}] attempted to directly perform [{}] against the security index [{}]", - authentication.getUser().principal(), action, SecurityLifecycleService.SECURITY_INDEX_NAME); + authentication.getUser().principal(), action, SecurityIndexManager.SECURITY_INDEX_NAME); throw denial(authentication, action, request, permission.names()); } else { putTransientIfNonExisting(AuthorizationServiceField.INDICES_PERMISSIONS_KEY, indicesAccessControl); @@ -337,7 +337,7 @@ && isSuperuser(authentication.getUser()) == false) { } private boolean hasSecurityIndexAccess(IndicesAccessControl indicesAccessControl) { - for (String index : SecurityLifecycleService.indexNames()) { + for (String index : SecurityIndexManager.indexNames()) { final IndicesAccessControl.IndexAccessControl indexPermissions = indicesAccessControl.getIndexPermissions(index); if (indexPermissions != null && indexPermissions.isGranted()) { return true; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizedIndices.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizedIndices.java index 3f257b7f0ce91..3068a3993d309 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizedIndices.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizedIndices.java @@ -9,7 +9,7 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.xpack.core.security.authz.permission.Role; import org.elasticsearch.xpack.core.security.user.User; -import org.elasticsearch.xpack.security.SecurityLifecycleService; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import java.util.ArrayList; import java.util.Collections; @@ -58,7 +58,7 @@ private List load() { if (isSuperuser(user) == false) { // we should filter out all of the security indices from wildcards - indicesAndAliases.removeAll(SecurityLifecycleService.indexNames()); + indicesAndAliases.removeAll(SecurityIndexManager.indexNames()); } return Collections.unmodifiableList(indicesAndAliases); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java index d90f50ca5faa5..b5a20af8d30b9 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java @@ -7,8 +7,6 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.cluster.health.ClusterIndexHealth; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; @@ -34,7 +32,6 @@ import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege; import org.elasticsearch.xpack.core.security.authz.privilege.Privilege; import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; -import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.support.SecurityIndexManager; import java.util.ArrayList; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java index 834a70b9e0304..b1e5170a2027c 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java @@ -43,7 +43,7 @@ import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges; import org.elasticsearch.xpack.core.security.client.SecurityClient; -import org.elasticsearch.xpack.security.SecurityLifecycleService; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import java.io.IOException; import java.util.ArrayList; @@ -85,22 +85,21 @@ public class NativeRolesStore extends AbstractComponent { private final XPackLicenseState licenseState; private SecurityClient securityClient; - private final SecurityLifecycleService securityLifecycleService; + private final SecurityIndexManager securityIndex; - public NativeRolesStore(Settings settings, Client client, XPackLicenseState licenseState, - SecurityLifecycleService securityLifecycleService) { + public NativeRolesStore(Settings settings, Client client, XPackLicenseState licenseState, SecurityIndexManager securityIndex) { super(settings); this.client = client; this.securityClient = new SecurityClient(client); this.licenseState = licenseState; - this.securityLifecycleService = securityLifecycleService; + this.securityIndex = securityIndex; } /** * Retrieve a list of roles, if rolesToGet is null or empty, fetch all roles */ public void getRoleDescriptors(String[] names, final ActionListener> listener) { - if (securityLifecycleService.securityIndex().indexExists() == false) { + if (securityIndex.indexExists() == false) { // TODO remove this short circuiting and fix tests that fail without this! listener.onResponse(Collections.emptyList()); } else if (names != null && names.length == 1) { @@ -108,7 +107,7 @@ public void getRoleDescriptors(String[] names, final ActionListener { + securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> { QueryBuilder query; if (names == null || names.length == 0) { query = QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE); @@ -118,7 +117,7 @@ public void getRoleDescriptors(String[] names, final ActionListener supplier = client.threadPool().getThreadContext().newRestorableContext(false); try (ThreadContext.StoredContext ignore = stashWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN)) { - SearchRequest request = client.prepareSearch(SecurityLifecycleService.SECURITY_INDEX_NAME) + SearchRequest request = client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) .setScroll(TimeValue.timeValueSeconds(10L)) .setQuery(query) .setSize(1000) @@ -133,8 +132,8 @@ public void getRoleDescriptors(String[] names, final ActionListener listener) { - securityLifecycleService.securityIndex().prepareIndexIfNeededThenExecute(listener::onFailure, () -> { - DeleteRequest request = client.prepareDelete(SecurityLifecycleService.SECURITY_INDEX_NAME, + securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> { + DeleteRequest request = client.prepareDelete(SecurityIndexManager.SECURITY_INDEX_NAME, ROLE_DOC_TYPE, getIdForUser(deleteRoleRequest.name())).request(); request.setRefreshPolicy(deleteRoleRequest.getRefreshPolicy()); executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, request, @@ -166,7 +165,7 @@ public void putRole(final PutRoleRequest request, final RoleDescriptor role, fin // pkg-private for testing void innerPutRole(final PutRoleRequest request, final RoleDescriptor role, final ActionListener listener) { - securityLifecycleService.securityIndex().prepareIndexIfNeededThenExecute(listener::onFailure, () -> { + securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> { final XContentBuilder xContentBuilder; try { xContentBuilder = role.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS, true); @@ -175,7 +174,7 @@ void innerPutRole(final PutRoleRequest request, final RoleDescriptor role, final return; } executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareIndex(SecurityLifecycleService.SECURITY_INDEX_NAME, ROLE_DOC_TYPE, getIdForUser(role.getName())) + client.prepareIndex(SecurityIndexManager.SECURITY_INDEX_NAME, ROLE_DOC_TYPE, getIdForUser(role.getName())) .setSource(xContentBuilder) .setRefreshPolicy(request.getRefreshPolicy()) .request(), @@ -197,19 +196,19 @@ public void onFailure(Exception e) { public void usageStats(ActionListener> listener) { Map usageStats = new HashMap<>(); - if (securityLifecycleService.securityIndex().indexExists() == false) { + if (securityIndex.indexExists() == false) { usageStats.put("size", 0L); usageStats.put("fls", false); usageStats.put("dls", false); listener.onResponse(usageStats); } else { - securityLifecycleService.securityIndex().prepareIndexIfNeededThenExecute(listener::onFailure, () -> + securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, client.prepareMultiSearch() - .add(client.prepareSearch(SecurityLifecycleService.SECURITY_INDEX_NAME) + .add(client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) .setQuery(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE)) .setSize(0)) - .add(client.prepareSearch(SecurityLifecycleService.SECURITY_INDEX_NAME) + .add(client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) .setQuery(QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE)) .must(QueryBuilders.boolQuery() @@ -219,7 +218,7 @@ public void usageStats(ActionListener> listener) { .should(existsQuery("indices.fields")))) .setSize(0) .setTerminateAfter(1)) - .add(client.prepareSearch(SecurityLifecycleService.SECURITY_INDEX_NAME) + .add(client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) .setQuery(QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE)) .filter(existsQuery("indices.query"))) @@ -259,11 +258,11 @@ public void onFailure(Exception e) { } private void getRoleDescriptor(final String roleId, ActionListener roleActionListener) { - if (securityLifecycleService.securityIndex().indexExists() == false) { + if (securityIndex.indexExists() == false) { // TODO remove this short circuiting and fix tests that fail without this! roleActionListener.onResponse(null); } else { - securityLifecycleService.securityIndex().prepareIndexIfNeededThenExecute(roleActionListener::onFailure, () -> + securityIndex.prepareIndexIfNeededThenExecute(roleActionListener::onFailure, () -> executeGetRoleRequest(roleId, new ActionListener() { @Override public void onResponse(GetResponse response) { @@ -288,9 +287,9 @@ public void onFailure(Exception e) { } private void executeGetRoleRequest(String role, ActionListener listener) { - securityLifecycleService.securityIndex().prepareIndexIfNeededThenExecute(listener::onFailure, () -> + securityIndex.prepareIndexIfNeededThenExecute(listener::onFailure, () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, - client.prepareGet(SecurityLifecycleService.SECURITY_INDEX_NAME, + client.prepareGet(SecurityIndexManager.SECURITY_INDEX_NAME, ROLE_DOC_TYPE, getIdForUser(role)).request(), listener, client::get)); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index 4bcfb779b0d50..45c55c633d923 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -23,6 +23,7 @@ import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.health.ClusterIndexHealth; import org.elasticsearch.cluster.metadata.AliasOrIndex; @@ -30,15 +31,19 @@ import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.xpack.core.template.TemplateUtils; import org.elasticsearch.xpack.core.upgrade.IndexUpgradeCheckVersion; import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -54,18 +59,18 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_FORMAT_SETTING; import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; -import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_INDEX_NAME; /** * Manages the lifecycle of a single index, its template, mapping and and data upgrades/migrations. */ -public class SecurityIndexManager extends AbstractComponent { +public class SecurityIndexManager extends AbstractComponent implements ClusterStateListener { public static final String INTERNAL_SECURITY_INDEX = ".security-" + IndexUpgradeCheckVersion.UPRADE_VERSION; public static final int INTERNAL_INDEX_FORMAT = 6; public static final String SECURITY_VERSION_STRING = "security-version"; public static final String TEMPLATE_VERSION_PATTERN = Pattern.quote("${security.template.version}"); public static final String SECURITY_TEMPLATE_NAME = "security-index-template"; + public static final String SECURITY_INDEX_NAME = ".security"; private final String indexName; private final Client client; @@ -74,10 +79,15 @@ public class SecurityIndexManager extends AbstractComponent { private volatile State indexState = new State(false, false, false, false, null, null); - public SecurityIndexManager(Settings settings, Client client, String indexName) { + public SecurityIndexManager(Settings settings, Client client, String indexName, ClusterService clusterService) { super(settings); this.client = client; this.indexName = indexName; + clusterService.addListener(this); + } + + public static List indexNames() { + return Collections.unmodifiableList(Arrays.asList(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX)); } public boolean checkMappingVersion(Predicate requiredVersion) { @@ -115,7 +125,14 @@ public void addIndexStateListener(BiConsumer listener) { stateChangeListeners.add(listener); } + @Override public void clusterChanged(ClusterChangedEvent event) { + if (event.state().blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) { + // wait until the gateway has recovered from disk, otherwise we think we don't have the + // .security index but they may not have been restored from the cluster state on disk + logger.debug("security index manager waiting until state has been recovered"); + return; + } final State previousState = indexState; final IndexMetaData indexMetaData = resolveConcreteIndex(indexName, event.state().metaData()); final boolean indexExists = indexMetaData != null; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java index 3c3eddfc14c08..d269de25c612d 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java @@ -7,16 +7,14 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.Client; -import org.elasticsearch.common.network.NetworkModule; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.NativeRealmIntegTestCase; import org.elasticsearch.xpack.core.security.action.role.DeleteRoleResponse; import org.elasticsearch.xpack.core.security.action.role.GetRolesResponse; import org.elasticsearch.xpack.core.security.action.role.PutRoleResponse; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.client.SecurityClient; -import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.authz.store.NativeRolesStore; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.junit.Before; import org.junit.BeforeClass; @@ -57,7 +55,7 @@ public void setupForTests() { logger.debug("--> created role [{}]", role); } - ensureGreen(SecurityLifecycleService.SECURITY_INDEX_NAME); + ensureGreen(SecurityIndexManager.SECURITY_INDEX_NAME); // warm up the caches on every node for (NativeRolesStore rolesStore : internalCluster().getInstances(NativeRolesStore.class)) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java index bc19df6185d63..00b46b332cb7c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java @@ -64,7 +64,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; -import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; import static org.hamcrest.Matchers.is; import static org.hamcrest.core.IsCollectionContaining.hasItem; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java index 988f60fe57e4f..190c8703955b1 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java @@ -62,7 +62,7 @@ import java.util.function.Predicate; import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_FORMAT_SETTING; -import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_INDEX_FORMAT; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java index 52a2e537d8db5..09a48a0eb1370 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java @@ -57,7 +57,6 @@ import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings; import org.elasticsearch.xpack.core.security.authc.saml.SamlRealmSettings; import org.elasticsearch.xpack.core.security.user.User; -import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.authc.Realms; import org.elasticsearch.xpack.security.authc.TokenService; import org.elasticsearch.xpack.security.authc.UserToken; @@ -161,16 +160,14 @@ void doExecute(Action action, Request request } }; - final SecurityLifecycleService lifecycleService = mock(SecurityLifecycleService.class); final SecurityIndexManager securityIndex = mock(SecurityIndexManager.class); - when(lifecycleService.securityIndex()).thenReturn(securityIndex); doAnswer(inv -> { ((Runnable) inv.getArguments()[1]).run(); return null; }).when(securityIndex).prepareIndexIfNeededThenExecute(any(Consumer.class), any(Runnable.class)); final ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool); - tokenService = new TokenService(settings, Clock.systemUTC(), client, lifecycleService, clusterService); + tokenService = new TokenService(settings, Clock.systemUTC(), client, securityIndex, clusterService); final TransportService transportService = new TransportService(Settings.EMPTY, null, null, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, Collections.emptySet()); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutActionTests.java index 93e6ebf2861cf..eca52831d9adc 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutActionTests.java @@ -47,7 +47,6 @@ import org.elasticsearch.xpack.core.security.authc.saml.SamlRealmSettings; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.core.ssl.SSLService; -import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.authc.Realms; import org.elasticsearch.xpack.security.authc.TokenService; import org.elasticsearch.xpack.security.authc.UserToken; @@ -173,16 +172,14 @@ public void setup() throws Exception { return Void.TYPE; }).when(client).execute(eq(IndexAction.INSTANCE), any(IndexRequest.class), any(ActionListener.class)); - final SecurityLifecycleService lifecycleService = mock(SecurityLifecycleService.class); final SecurityIndexManager securityIndex = mock(SecurityIndexManager.class); - when(lifecycleService.securityIndex()).thenReturn(securityIndex); doAnswer(inv -> { ((Runnable) inv.getArguments()[1]).run(); return null; }).when(securityIndex).prepareIndexIfNeededThenExecute(any(Consumer.class), any(Runnable.class)); final ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool); - tokenService = new TokenService(settings, Clock.systemUTC(), client, lifecycleService, clusterService); + tokenService = new TokenService(settings, Clock.systemUTC(), client, securityIndex, clusterService); final TransportService transportService = new TransportService(Settings.EMPTY, null, null, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, Collections.emptySet()); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersActionTests.java index 02af431f8978b..6750560b0b0d2 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersActionTests.java @@ -24,7 +24,6 @@ import org.elasticsearch.xpack.core.security.user.SystemUser; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.core.security.user.XPackUser; -import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore; import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm; import org.elasticsearch.xpack.security.authc.esnative.ReservedRealmTests; @@ -76,13 +75,11 @@ public void maybeEnableAnonymous() { public void testAnonymousUser() { NativeUsersStore usersStore = mock(NativeUsersStore.class); - SecurityLifecycleService securityLifecycleService = mock(SecurityLifecycleService.class); SecurityIndexManager securityIndex = mock(SecurityIndexManager.class); - when(securityLifecycleService.securityIndex()).thenReturn(securityIndex); when(securityIndex.isAvailable()).thenReturn(true); AnonymousUser anonymousUser = new AnonymousUser(settings); ReservedRealm reservedRealm = - new ReservedRealm(mock(Environment.class), settings, usersStore, anonymousUser, securityLifecycleService, new ThreadContext(Settings.EMPTY)); + new ReservedRealm(mock(Environment.class), settings, usersStore, anonymousUser, securityIndex, new ThreadContext(Settings.EMPTY)); TransportService transportService = new TransportService(Settings.EMPTY, null, null, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, Collections.emptySet()); TransportGetUsersAction action = new TransportGetUsersAction(Settings.EMPTY, mock(ThreadPool.class), mock(ActionFilters.class), @@ -148,15 +145,13 @@ public void onFailure(Exception e) { public void testReservedUsersOnly() { NativeUsersStore usersStore = mock(NativeUsersStore.class); - SecurityLifecycleService securityLifecycleService = mock(SecurityLifecycleService.class); SecurityIndexManager securityIndex = mock(SecurityIndexManager.class); - when(securityLifecycleService.securityIndex()).thenReturn(securityIndex); when(securityIndex.isAvailable()).thenReturn(true); when(securityIndex.checkMappingVersion(any())).thenReturn(true); ReservedRealmTests.mockGetAllReservedUserInfo(usersStore, Collections.emptyMap()); ReservedRealm reservedRealm = - new ReservedRealm(mock(Environment.class), settings, usersStore, new AnonymousUser(settings), securityLifecycleService, new ThreadContext(Settings.EMPTY)); + new ReservedRealm(mock(Environment.class), settings, usersStore, new AnonymousUser(settings), securityIndex, new ThreadContext(Settings.EMPTY)); PlainActionFuture> userFuture = new PlainActionFuture<>(); reservedRealm.users(userFuture); final Collection allReservedUsers = userFuture.actionGet(); @@ -198,13 +193,11 @@ public void testGetAllUsers() { final List storeUsers = randomFrom(Collections.emptyList(), Collections.singletonList(new User("joe")), Arrays.asList(new User("jane"), new User("fred")), randomUsers()); NativeUsersStore usersStore = mock(NativeUsersStore.class); - SecurityLifecycleService securityLifecycleService = mock(SecurityLifecycleService.class); SecurityIndexManager securityIndex = mock(SecurityIndexManager.class); - when(securityLifecycleService.securityIndex()).thenReturn(securityIndex); when(securityIndex.isAvailable()).thenReturn(true); ReservedRealmTests.mockGetAllReservedUserInfo(usersStore, Collections.emptyMap()); ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, new AnonymousUser(settings), - securityLifecycleService, new ThreadContext(Settings.EMPTY)); + securityIndex, new ThreadContext(Settings.EMPTY)); TransportService transportService = new TransportService(Settings.EMPTY, null, null, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, Collections.emptySet()); TransportGetUsersAction action = new TransportGetUsersAction(Settings.EMPTY, mock(ThreadPool.class), mock(ActionFilters.class), diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportPutUserActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportPutUserActionTests.java index 7b26e605207a2..65cf74971a55c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportPutUserActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportPutUserActionTests.java @@ -25,7 +25,6 @@ import org.elasticsearch.xpack.core.security.user.SystemUser; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.core.security.user.XPackUser; -import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore; import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm; import org.elasticsearch.xpack.security.authc.esnative.ReservedRealmTests; @@ -118,14 +117,12 @@ public void onFailure(Exception e) { public void testReservedUser() { NativeUsersStore usersStore = mock(NativeUsersStore.class); - SecurityLifecycleService securityLifecycleService = mock(SecurityLifecycleService.class); SecurityIndexManager securityIndex = mock(SecurityIndexManager.class); - when(securityLifecycleService.securityIndex()).thenReturn(securityIndex); when(securityIndex.isAvailable()).thenReturn(true); ReservedRealmTests.mockGetAllReservedUserInfo(usersStore, Collections.emptyMap()); Settings settings = Settings.builder().put("path.home", createTempDir()).build(); ReservedRealm reservedRealm = new ReservedRealm(TestEnvironment.newEnvironment(settings), settings, usersStore, - new AnonymousUser(settings), securityLifecycleService, new ThreadContext(settings)); + new AnonymousUser(settings), securityIndex, new ThreadContext(settings)); PlainActionFuture> userFuture = new PlainActionFuture<>(); reservedRealm.users(userFuture); final User reserved = randomFrom(userFuture.actionGet().toArray(new User[0])); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java index 41b765cb33322..cd685b8f34c28 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java @@ -64,7 +64,6 @@ import org.elasticsearch.xpack.core.security.user.AnonymousUser; import org.elasticsearch.xpack.core.security.user.SystemUser; import org.elasticsearch.xpack.core.security.user.User; -import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.audit.AuditTrailService; import org.elasticsearch.xpack.security.authc.AuthenticationService.Authenticator; import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm; @@ -125,7 +124,6 @@ public class AuthenticationServiceTests extends ESTestCase { private ThreadPool threadPool; private ThreadContext threadContext; private TokenService tokenService; - private SecurityLifecycleService lifecycleService; private SecurityIndexManager securityIndex; private Client client; private InetSocketAddress remoteAddress; @@ -182,16 +180,14 @@ licenseState, threadContext, mock(ReservedRealm.class), Arrays.asList(firstRealm .setId((String) invocationOnMock.getArguments()[2]); return builder; }).when(client).prepareGet(anyString(), anyString(), anyString()); - lifecycleService = mock(SecurityLifecycleService.class); securityIndex = mock(SecurityIndexManager.class); - when(lifecycleService.securityIndex()).thenReturn(securityIndex); doAnswer(invocationOnMock -> { Runnable runnable = (Runnable) invocationOnMock.getArguments()[1]; runnable.run(); return null; }).when(securityIndex).prepareIndexIfNeededThenExecute(any(Consumer.class), any(Runnable.class)); ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool); - tokenService = new TokenService(settings, Clock.systemUTC(), client, lifecycleService, clusterService); + tokenService = new TokenService(settings, Clock.systemUTC(), client, securityIndex, clusterService); service = new AuthenticationService(settings, realms, auditTrail, new DefaultAuthenticationFailureHandler(), threadPool, new AnonymousUser(settings), tokenService); } @@ -929,7 +925,7 @@ public void testInvalidToken() throws Exception { public void testExpiredToken() throws Exception { when(securityIndex.isAvailable()).thenReturn(true); - when(lifecycleService.securityIndex().indexExists()).thenReturn(true); + when(securityIndex.indexExists()).thenReturn(true); User user = new User("_username", "r1"); final Authentication expected = new Authentication(user, new RealmRef("realm", "custom", "node"), null); PlainActionFuture> tokenFuture = new PlainActionFuture<>(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/InternalRealmsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/InternalRealmsTests.java index 47eb1eabae159..0cbeced00b2ab 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/InternalRealmsTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/InternalRealmsTests.java @@ -15,7 +15,6 @@ import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings; import org.elasticsearch.xpack.core.ssl.SSLService; -import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore; import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore; import org.elasticsearch.xpack.security.support.SecurityIndexManager; @@ -31,18 +30,15 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyZeroInteractions; -import static org.mockito.Mockito.when; public class InternalRealmsTests extends ESTestCase { public void testNativeRealmRegistersIndexHealthChangeListener() throws Exception { - SecurityLifecycleService lifecycleService = mock(SecurityLifecycleService.class); SecurityIndexManager securityIndex = mock(SecurityIndexManager.class); - when(lifecycleService.securityIndex()).thenReturn(securityIndex); Map factories = InternalRealms.getFactories(mock(ThreadPool.class), mock(ResourceWatcherService.class), - mock(SSLService.class), mock(NativeUsersStore.class), mock(NativeRoleMappingStore.class), lifecycleService); + mock(SSLService.class), mock(NativeUsersStore.class), mock(NativeRoleMappingStore.class), securityIndex); assertThat(factories, hasEntry(is(NativeRealmSettings.TYPE), any(Realm.Factory.class))); - verifyZeroInteractions(lifecycleService); + verifyZeroInteractions(securityIndex); Settings settings = Settings.builder().put("path.home", createTempDir()).build(); factories.get(NativeRealmSettings.TYPE).create(new RealmConfig("test", Settings.EMPTY, settings, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java index a8a0f858d9c03..ec4a97b7f392b 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java @@ -32,7 +32,7 @@ import org.elasticsearch.xpack.core.security.authc.TokenMetaData; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.client.SecurityClient; -import org.elasticsearch.xpack.security.SecurityLifecycleService; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.junit.After; import org.junit.Before; @@ -147,7 +147,7 @@ public void testExpiredTokensDeletedAfterExpiration() throws Exception { assertTrue(invalidateResponse.isCreated()); AtomicReference docId = new AtomicReference<>(); assertBusy(() -> { - SearchResponse searchResponse = client.prepareSearch(SecurityLifecycleService.SECURITY_INDEX_NAME) + SearchResponse searchResponse = client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) .setSource(SearchSourceBuilder.searchSource() .query(QueryBuilders.termQuery("doc_type", TokenService.INVALIDATED_TOKEN_DOC_TYPE))) .setSize(1) @@ -160,7 +160,7 @@ public void testExpiredTokensDeletedAfterExpiration() throws Exception { // hack doc to modify the time to the day before Instant dayBefore = created.minus(1L, ChronoUnit.DAYS); assertTrue(Instant.now().isAfter(dayBefore)); - client.prepareUpdate(SecurityLifecycleService.SECURITY_INDEX_NAME, "doc", docId.get()) + client.prepareUpdate(SecurityIndexManager.SECURITY_INDEX_NAME, "doc", docId.get()) .setDoc("expiration_time", dayBefore.toEpochMilli()) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); @@ -178,8 +178,8 @@ public void testExpiredTokensDeletedAfterExpiration() throws Exception { assertEquals("token malformed", e.getMessage()); } } - client.admin().indices().prepareRefresh(SecurityLifecycleService.SECURITY_INDEX_NAME).get(); - SearchResponse searchResponse = client.prepareSearch(SecurityLifecycleService.SECURITY_INDEX_NAME) + client.admin().indices().prepareRefresh(SecurityIndexManager.SECURITY_INDEX_NAME).get(); + SearchResponse searchResponse = client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME) .setSource(SearchSourceBuilder.searchSource() .query(QueryBuilders.termQuery("doc_type", TokenService.INVALIDATED_TOKEN_DOC_TYPE))) .setSize(0) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java index 79a2647997505..af1034d957455 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java @@ -50,7 +50,6 @@ import org.elasticsearch.xpack.core.security.authc.TokenMetaData; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.core.watcher.watch.ClockMock; -import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.junit.AfterClass; import org.junit.Before; @@ -86,7 +85,6 @@ public class TokenServiceTests extends ESTestCase { .put(XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.getKey(), true).build(); private Client client; - private SecurityLifecycleService lifecycleService; private SecurityIndexManager securityIndex; private ClusterService clusterService; private Settings tokenServiceEnabledSettings = Settings.builder() @@ -132,9 +130,7 @@ public void setupClient() { }).when(client).execute(eq(IndexAction.INSTANCE), any(IndexRequest.class), any(ActionListener.class)); // setup lifecycle service - lifecycleService = mock(SecurityLifecycleService.class); securityIndex = mock(SecurityIndexManager.class); - when(lifecycleService.securityIndex()).thenReturn(securityIndex); doAnswer(invocationOnMock -> { Runnable runnable = (Runnable) invocationOnMock.getArguments()[1]; runnable.run(); @@ -157,7 +153,7 @@ public static void shutdownThreadpool() throws InterruptedException { } public void testAttachAndGetToken() throws Exception { - TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, lifecycleService, clusterService); + TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex, clusterService); Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null); PlainActionFuture> tokenFuture = new PlainActionFuture<>(); tokenService.createUserToken(authentication, authentication, tokenFuture, Collections.emptyMap()); @@ -177,7 +173,7 @@ public void testAttachAndGetToken() throws Exception { try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) { // verify a second separate token service with its own salt can also verify - TokenService anotherService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, lifecycleService + TokenService anotherService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex , clusterService); anotherService.refreshMetaData(tokenService.getTokenMetaData()); PlainActionFuture future = new PlainActionFuture<>(); @@ -188,7 +184,7 @@ public void testAttachAndGetToken() throws Exception { } public void testRotateKey() throws Exception { - TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, lifecycleService, clusterService); + TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex, clusterService); Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null); PlainActionFuture> tokenFuture = new PlainActionFuture<>(); tokenService.createUserToken(authentication, authentication, tokenFuture, Collections.emptyMap()); @@ -240,12 +236,12 @@ private void rotateKeys(TokenService tokenService) { } public void testKeyExchange() throws Exception { - TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, lifecycleService, clusterService); + TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex, clusterService); int numRotations = 0;randomIntBetween(1, 5); for (int i = 0; i < numRotations; i++) { rotateKeys(tokenService); } - TokenService otherTokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, lifecycleService, + TokenService otherTokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex, clusterService); otherTokenService.refreshMetaData(tokenService.getTokenMetaData()); Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null); @@ -277,7 +273,7 @@ public void testKeyExchange() throws Exception { } public void testPruneKeys() throws Exception { - TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, lifecycleService, clusterService); + TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex, clusterService); Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null); PlainActionFuture> tokenFuture = new PlainActionFuture<>(); tokenService.createUserToken(authentication, authentication, tokenFuture, Collections.emptyMap()); @@ -338,7 +334,7 @@ public void testPruneKeys() throws Exception { } public void testPassphraseWorks() throws Exception { - TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, lifecycleService, clusterService); + TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex, clusterService); Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null); PlainActionFuture> tokenFuture = new PlainActionFuture<>(); tokenService.createUserToken(authentication, authentication, tokenFuture, Collections.emptyMap()); @@ -358,7 +354,7 @@ public void testPassphraseWorks() throws Exception { try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) { // verify a second separate token service with its own passphrase cannot verify - TokenService anotherService = new TokenService(Settings.EMPTY, systemUTC(), client, lifecycleService, + TokenService anotherService = new TokenService(Settings.EMPTY, systemUTC(), client, securityIndex, clusterService); PlainActionFuture future = new PlainActionFuture<>(); anotherService.getAndValidateToken(requestContext, future); @@ -367,7 +363,7 @@ public void testPassphraseWorks() throws Exception { } public void testGetTokenWhenKeyCacheHasExpired() throws Exception { - TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, lifecycleService, clusterService); + TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex, clusterService); Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null); PlainActionFuture> tokenFuture = new PlainActionFuture<>(); @@ -382,7 +378,7 @@ public void testGetTokenWhenKeyCacheHasExpired() throws Exception { public void testInvalidatedToken() throws Exception { when(securityIndex.indexExists()).thenReturn(true); TokenService tokenService = - new TokenService(tokenServiceEnabledSettings, systemUTC(), client, lifecycleService, clusterService); + new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex, clusterService); Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null); PlainActionFuture> tokenFuture = new PlainActionFuture<>(); tokenService.createUserToken(authentication, authentication, tokenFuture, Collections.emptyMap()); @@ -436,7 +432,7 @@ public void testComputeSecretKeyIsConsistent() throws Exception { public void testTokenExpiry() throws Exception { ClockMock clock = ClockMock.frozen(); - TokenService tokenService = new TokenService(tokenServiceEnabledSettings, clock, client, lifecycleService, clusterService); + TokenService tokenService = new TokenService(tokenServiceEnabledSettings, clock, client, securityIndex, clusterService); Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null); PlainActionFuture> tokenFuture = new PlainActionFuture<>(); tokenService.createUserToken(authentication, authentication, tokenFuture, Collections.emptyMap()); @@ -488,7 +484,7 @@ public void testTokenServiceDisabled() throws Exception { TokenService tokenService = new TokenService(Settings.builder() .put(XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.getKey(), false) .build(), - Clock.systemUTC(), client, lifecycleService, clusterService); + Clock.systemUTC(), client, securityIndex, clusterService); IllegalStateException e = expectThrows(IllegalStateException.class, () -> tokenService.createUserToken(null, null, null, null)); assertEquals("tokens are not enabled", e.getMessage()); @@ -530,7 +526,7 @@ public void testMalformedToken() throws Exception { final int numBytes = randomIntBetween(1, TokenService.MINIMUM_BYTES + 32); final byte[] randomBytes = new byte[numBytes]; random().nextBytes(randomBytes); - TokenService tokenService = new TokenService(Settings.EMPTY, systemUTC(), client, lifecycleService, clusterService); + TokenService tokenService = new TokenService(Settings.EMPTY, systemUTC(), client, securityIndex, clusterService); ThreadContext requestContext = new ThreadContext(Settings.EMPTY); requestContext.putHeader("Authorization", "Bearer " + Base64.getEncoder().encodeToString(randomBytes)); @@ -544,7 +540,7 @@ public void testMalformedToken() throws Exception { public void testIndexNotAvailable() throws Exception { TokenService tokenService = - new TokenService(tokenServiceEnabledSettings, systemUTC(), client, lifecycleService, clusterService); + new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex, clusterService); Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null); PlainActionFuture> tokenFuture = new PlainActionFuture<>(); tokenService.createUserToken(authentication, authentication, tokenFuture, Collections.emptyMap()); @@ -577,7 +573,7 @@ public void testIndexNotAvailable() throws Exception { public void testGetAuthenticationWorksWithExpiredToken() throws Exception { TokenService tokenService = - new TokenService(tokenServiceEnabledSettings, Clock.systemUTC(), client, lifecycleService, clusterService); + new TokenService(tokenServiceEnabledSettings, Clock.systemUTC(), client, securityIndex, clusterService); Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null); UserToken expired = new UserToken(authentication, Instant.now().minus(3L, ChronoUnit.DAYS)); mockGetTokenFromId(expired); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java index 839b272d115cc..58cb515081a59 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java @@ -9,14 +9,13 @@ import joptsimple.OptionSet; import org.elasticsearch.cli.MockTerminal; import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.test.NativeRealmIntegTestCase; import org.elasticsearch.test.SecuritySettingsSource; import org.elasticsearch.xpack.core.security.authc.support.CharArrays; import org.elasticsearch.xpack.core.security.client.SecurityClient; -import org.elasticsearch.xpack.security.SecurityLifecycleService; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.junit.BeforeClass; import java.nio.charset.StandardCharsets; @@ -82,7 +81,7 @@ public void testRetrieveUsers() throws Exception { addedUsers.add(uname); } logger.error("--> waiting for .security index"); - ensureGreen(SecurityLifecycleService.SECURITY_INDEX_NAME); + ensureGreen(SecurityIndexManager.SECURITY_INDEX_NAME); MockTerminal t = new MockTerminal(); String username = nodeClientUsername(); @@ -127,7 +126,7 @@ public void testRetrieveRoles() throws Exception { addedRoles.add(rname); } logger.error("--> waiting for .security index"); - ensureGreen(SecurityLifecycleService.SECURITY_INDEX_NAME); + ensureGreen(SecurityIndexManager.SECURITY_INDEX_NAME); MockTerminal t = new MockTerminal(); String username = nodeClientUsername(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java index 2c11411955a0f..a238576e41323 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java @@ -54,7 +54,7 @@ import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; -import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.INTERNAL_SECURITY_INDEX; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.containsString; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java index 091f6f2ed4571..59244bbc50975 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.xpack.core.security.user.KibanaUser; import org.elasticsearch.xpack.core.security.user.LogstashSystemUser; import org.elasticsearch.xpack.core.security.user.User; -import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.junit.Before; @@ -113,7 +112,7 @@ public void testBlankPasswordInIndexImpliesDefaultPassword() throws Exception { values.put(PASSWORD_FIELD, BLANK_PASSWORD); final GetResult result = new GetResult( - SecurityLifecycleService.SECURITY_INDEX_NAME, + SecurityIndexManager.SECURITY_INDEX_NAME, NativeUsersStore.INDEX_TYPE, NativeUsersStore.getIdForUser(NativeUsersStore.RESERVED_USER_TYPE, randomAlphaOfLength(12)), 1L, @@ -182,7 +181,7 @@ public void testVerifyNonExistentUser() throws Exception { nativeUsersStore.verifyPassword(username, password, future); final GetResult getResult = new GetResult( - SecurityLifecycleService.SECURITY_INDEX_NAME, + SecurityIndexManager.SECURITY_INDEX_NAME, NativeUsersStore.INDEX_TYPE, NativeUsersStore.getIdForUser(NativeUsersStore.USER_DOC_TYPE, username), 1L, @@ -223,7 +222,7 @@ private void respondToGetUserRequest(String username, SecureString password, Str values.put(User.Fields.TYPE.getPreferredName(), NativeUsersStore.USER_DOC_TYPE); final BytesReference source = BytesReference.bytes(jsonBuilder().map(values)); final GetResult getResult = new GetResult( - SecurityLifecycleService.SECURITY_INDEX_NAME, + SecurityIndexManager.SECURITY_INDEX_NAME, NativeUsersStore.INDEX_TYPE, NativeUsersStore.getIdForUser(NativeUsersStore.USER_DOC_TYPE, username), 1L, @@ -236,9 +235,7 @@ private void respondToGetUserRequest(String username, SecureString password, Str } private NativeUsersStore startNativeUsersStore() { - SecurityLifecycleService securityLifecycleService = mock(SecurityLifecycleService.class); SecurityIndexManager securityIndex = mock(SecurityIndexManager.class); - when(securityLifecycleService.securityIndex()).thenReturn(securityIndex); when(securityIndex.isAvailable()).thenReturn(true); when(securityIndex.indexExists()).thenReturn(true); when(securityIndex.isMappingUpToDate()).thenReturn(true); @@ -248,7 +245,7 @@ private NativeUsersStore startNativeUsersStore() { action.run(); return null; }).when(securityIndex).prepareIndexIfNeededThenExecute(any(Consumer.class), any(Runnable.class)); - return new NativeUsersStore(Settings.EMPTY, client, securityLifecycleService); + return new NativeUsersStore(Settings.EMPTY, client, securityIndex); } } \ No newline at end of file diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmTests.java index 024f8f603c928..9fc52e8af63bc 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmTests.java @@ -27,7 +27,6 @@ import org.elasticsearch.xpack.core.security.user.LogstashSystemUser; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.core.security.user.UsernamesField; -import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore.ReservedUserInfo; import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.junit.Before; @@ -63,15 +62,12 @@ public class ReservedRealmTests extends ESTestCase { private static final SecureString EMPTY_PASSWORD = new SecureString("".toCharArray()); private NativeUsersStore usersStore; - private SecurityLifecycleService securityLifecycleService; private SecurityIndexManager securityIndex; @Before public void setupMocks() throws Exception { usersStore = mock(NativeUsersStore.class); - securityLifecycleService = mock(SecurityLifecycleService.class); securityIndex = mock(SecurityIndexManager.class); - when(securityLifecycleService.securityIndex()).thenReturn(securityIndex); when(securityIndex.isAvailable()).thenReturn(true); when(securityIndex.checkMappingVersion(any())).thenReturn(true); mockGetAllReservedUserInfo(usersStore, Collections.emptyMap()); @@ -82,7 +78,7 @@ public void testReservedUserEmptyPasswordAuthenticationFails() throws Throwable UsernamesField.BEATS_NAME); final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore, - new AnonymousUser(Settings.EMPTY), securityLifecycleService, new ThreadContext(Settings.EMPTY)); + new AnonymousUser(Settings.EMPTY), securityIndex, new ThreadContext(Settings.EMPTY)); PlainActionFuture listener = new PlainActionFuture<>(); @@ -98,7 +94,7 @@ public void testAuthenticationDisabled() throws Throwable { } final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, - new AnonymousUser(settings), securityLifecycleService, new ThreadContext(Settings.EMPTY)); + new AnonymousUser(settings), securityIndex, new ThreadContext(Settings.EMPTY)); final User expected = randomReservedUser(true); final String principal = expected.principal(); @@ -120,7 +116,7 @@ public void testAuthenticationDisabledUserWithStoredPassword() throws Throwable private void verifySuccessfulAuthentication(boolean enabled) throws Exception { final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore, - new AnonymousUser(Settings.EMPTY), securityLifecycleService, new ThreadContext(Settings.EMPTY)); + new AnonymousUser(Settings.EMPTY), securityIndex, new ThreadContext(Settings.EMPTY)); final User expectedUser = randomReservedUser(enabled); final String principal = expectedUser.principal(); final SecureString newPassword = new SecureString("foobar".toCharArray()); @@ -161,7 +157,7 @@ private void verifySuccessfulAuthentication(boolean enabled) throws Exception { public void testLookup() throws Exception { final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore, - new AnonymousUser(Settings.EMPTY), securityLifecycleService, new ThreadContext(Settings.EMPTY)); + new AnonymousUser(Settings.EMPTY), securityIndex, new ThreadContext(Settings.EMPTY)); final User expectedUser = randomReservedUser(true); final String principal = expectedUser.principal(); @@ -186,7 +182,7 @@ public void testLookupDisabled() throws Exception { Settings settings = Settings.builder().put(XPackSettings.RESERVED_REALM_ENABLED_SETTING.getKey(), false).build(); final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, new AnonymousUser(settings), - securityLifecycleService, new ThreadContext(Settings.EMPTY)); + securityIndex, new ThreadContext(Settings.EMPTY)); final User expectedUser = randomReservedUser(true); final String principal = expectedUser.principal(); @@ -200,7 +196,7 @@ public void testLookupDisabled() throws Exception { public void testLookupThrows() throws Exception { final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore, - new AnonymousUser(Settings.EMPTY), securityLifecycleService, new ThreadContext(Settings.EMPTY)); + new AnonymousUser(Settings.EMPTY), securityIndex, new ThreadContext(Settings.EMPTY)); final User expectedUser = randomReservedUser(true); final String principal = expectedUser.principal(); when(securityIndex.indexExists()).thenReturn(true); @@ -247,7 +243,7 @@ public void testIsReservedDisabled() { public void testGetUsers() { final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore, - new AnonymousUser(Settings.EMPTY), securityLifecycleService, new ThreadContext(Settings.EMPTY)); + new AnonymousUser(Settings.EMPTY), securityIndex, new ThreadContext(Settings.EMPTY)); PlainActionFuture> userFuture = new PlainActionFuture<>(); reservedRealm.users(userFuture); assertThat(userFuture.actionGet(), @@ -262,7 +258,7 @@ public void testGetUsersDisabled() { .build(); final AnonymousUser anonymousUser = new AnonymousUser(settings); final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, anonymousUser, - securityLifecycleService, new ThreadContext(Settings.EMPTY)); + securityIndex, new ThreadContext(Settings.EMPTY)); PlainActionFuture> userFuture = new PlainActionFuture<>(); reservedRealm.users(userFuture); if (anonymousEnabled) { @@ -279,7 +275,7 @@ public void testFailedAuthentication() throws Exception { ReservedUserInfo userInfo = new ReservedUserInfo(hash, true, false); mockGetAllReservedUserInfo(usersStore, Collections.singletonMap("elastic", userInfo)); final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore, - new AnonymousUser(Settings.EMPTY), securityLifecycleService, new ThreadContext(Settings.EMPTY)); + new AnonymousUser(Settings.EMPTY), securityIndex, new ThreadContext(Settings.EMPTY)); if (randomBoolean()) { PlainActionFuture future = new PlainActionFuture<>(); @@ -309,7 +305,7 @@ public void testBootstrapElasticPasswordWorksOnceSecurityIndexExists() throws Ex when(securityIndex.indexExists()).thenReturn(true); final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, - new AnonymousUser(Settings.EMPTY), securityLifecycleService, new ThreadContext(Settings.EMPTY)); + new AnonymousUser(Settings.EMPTY), securityIndex, new ThreadContext(Settings.EMPTY)); PlainActionFuture listener = new PlainActionFuture<>(); doAnswer((i) -> { @@ -331,7 +327,7 @@ public void testBootstrapElasticPasswordFailsOnceElasticUserExists() throws Exce when(securityIndex.indexExists()).thenReturn(true); final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, - new AnonymousUser(Settings.EMPTY), securityLifecycleService, new ThreadContext(Settings.EMPTY)); + new AnonymousUser(Settings.EMPTY), securityIndex, new ThreadContext(Settings.EMPTY)); PlainActionFuture listener = new PlainActionFuture<>(); SecureString password = new SecureString("password".toCharArray()); doAnswer((i) -> { @@ -358,7 +354,7 @@ public void testBootstrapElasticPasswordWorksBeforeSecurityIndexExists() throws when(securityIndex.indexExists()).thenReturn(false); final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, - new AnonymousUser(Settings.EMPTY), securityLifecycleService, new ThreadContext(Settings.EMPTY)); + new AnonymousUser(Settings.EMPTY), securityIndex, new ThreadContext(Settings.EMPTY)); PlainActionFuture listener = new PlainActionFuture<>(); reservedRealm.doAuthenticate(new UsernamePasswordToken(new ElasticUser(true).principal(), @@ -376,7 +372,7 @@ public void testNonElasticUsersCannotUseBootstrapPasswordWhenSecurityIndexExists when(securityIndex.indexExists()).thenReturn(true); final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, - new AnonymousUser(Settings.EMPTY), securityLifecycleService, new ThreadContext(Settings.EMPTY)); + new AnonymousUser(Settings.EMPTY), securityIndex, new ThreadContext(Settings.EMPTY)); PlainActionFuture listener = new PlainActionFuture<>(); final String principal = randomFrom(KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME); @@ -398,7 +394,7 @@ public void testNonElasticUsersCannotUseBootstrapPasswordWhenSecurityIndexDoesNo when(securityIndex.indexExists()).thenReturn(false); final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, - new AnonymousUser(Settings.EMPTY), securityLifecycleService, new ThreadContext(Settings.EMPTY)); + new AnonymousUser(Settings.EMPTY), securityIndex, new ThreadContext(Settings.EMPTY)); PlainActionFuture listener = new PlainActionFuture<>(); final String principal = randomFrom(KibanaUser.NAME, LogstashSystemUser.NAME, BeatsSystemUser.NAME); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java index 693118c21bde5..2a1c2dabe30b7 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java @@ -26,7 +26,6 @@ import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression; import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression.FieldValue; import org.elasticsearch.xpack.core.security.user.User; -import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm; import org.elasticsearch.xpack.security.authc.support.UserRoleMapper; import org.elasticsearch.xpack.security.support.SecurityIndexManager; @@ -73,12 +72,10 @@ public void testResolveRoles() throws Exception { Arrays.asList("mutants"), Collections.emptyMap(), false); final Client client = mock(Client.class); - final SecurityLifecycleService lifecycleService = mock(SecurityLifecycleService.class); SecurityIndexManager securityIndex = mock(SecurityIndexManager.class); - when(lifecycleService.securityIndex()).thenReturn(securityIndex); when(securityIndex.isAvailable()).thenReturn(true); - final NativeRoleMappingStore store = new NativeRoleMappingStore(Settings.EMPTY, client, lifecycleService) { + final NativeRoleMappingStore store = new NativeRoleMappingStore(Settings.EMPTY, client, securityIndex) { @Override protected void loadMappings(ActionListener> listener) { final List mappings = Arrays.asList(mapping1, mapping2, mapping3, mapping4); @@ -212,7 +209,7 @@ protected void doLookupUser(String username, ActionListener listener) { listener.onResponse(null); } }; - final NativeRoleMappingStore store = new NativeRoleMappingStore(Settings.EMPTY, client, mock(SecurityLifecycleService.class)); + final NativeRoleMappingStore store = new NativeRoleMappingStore(Settings.EMPTY, client, mock(SecurityIndexManager.class)); store.refreshRealmOnChange(mockRealm); return store; } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java index 3013a7c41c2ac..bcd31c32f7f78 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java @@ -140,7 +140,7 @@ import static org.elasticsearch.test.SecurityTestsUtils.assertAuthenticationException; import static org.elasticsearch.test.SecurityTestsUtils.assertThrowsAuthorizationException; import static org.elasticsearch.test.SecurityTestsUtils.assertThrowsAuthorizationExceptionRunAs; -import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.endsWith; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java index 4bb8af96ca8c8..1d0e5c179a9cd 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java @@ -20,8 +20,8 @@ import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege; import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege; import org.elasticsearch.xpack.core.security.user.User; -import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import java.util.List; @@ -81,7 +81,7 @@ public void testSecurityIndicesAreRemovedFromRegularUser() { MetaData metaData = MetaData.builder() .put(new IndexMetaData.Builder("an-index").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true) .put(new IndexMetaData.Builder("another-index").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true) - .put(new IndexMetaData.Builder(SecurityLifecycleService.SECURITY_INDEX_NAME).settings(indexSettings) + .put(new IndexMetaData.Builder(SecurityIndexManager.SECURITY_INDEX_NAME).settings(indexSettings) .numberOfShards(1).numberOfReplicas(0).build(), true) .build(); @@ -97,12 +97,12 @@ public void testSecurityIndicesAreNotRemovedFromSuperUsers() { MetaData metaData = MetaData.builder() .put(new IndexMetaData.Builder("an-index").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true) .put(new IndexMetaData.Builder("another-index").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true) - .put(new IndexMetaData.Builder(SecurityLifecycleService.SECURITY_INDEX_NAME).settings(indexSettings) + .put(new IndexMetaData.Builder(SecurityIndexManager.SECURITY_INDEX_NAME).settings(indexSettings) .numberOfShards(1).numberOfReplicas(0).build(), true) .build(); AuthorizedIndices authorizedIndices = new AuthorizedIndices(user, role, SearchAction.NAME, metaData); List list = authorizedIndices.get(); - assertThat(list, containsInAnyOrder("an-index", "another-index", SecurityLifecycleService.SECURITY_INDEX_NAME)); + assertThat(list, containsInAnyOrder("an-index", "another-index", SecurityIndexManager.SECURITY_INDEX_NAME)); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java index 17d8c754e1642..b080b5924ce7a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java @@ -61,10 +61,10 @@ import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.core.security.user.XPackSecurityUser; import org.elasticsearch.xpack.core.security.user.XPackUser; -import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.audit.AuditTrailService; import org.elasticsearch.xpack.security.authz.IndicesAndAliasesResolver.ResolvedIndices; import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.elasticsearch.xpack.security.test.SecurityTestUtils; import org.junit.Before; @@ -75,7 +75,7 @@ import java.util.Map; import java.util.Set; -import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -1199,14 +1199,14 @@ public void testXPackSecurityUserHasAccessToSecurityIndex() { { final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(XPackSecurityUser.INSTANCE, SearchAction.NAME); List indices = resolveIndices(request, authorizedIndices).getLocal(); - assertThat(indices, hasItem(SecurityLifecycleService.SECURITY_INDEX_NAME)); + assertThat(indices, hasItem(SecurityIndexManager.SECURITY_INDEX_NAME)); } { IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest(); aliasesRequest.addAliasAction(AliasActions.add().alias("security_alias").index(SECURITY_INDEX_NAME)); final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(XPackSecurityUser.INSTANCE, IndicesAliasesAction.NAME); List indices = resolveIndices(aliasesRequest, authorizedIndices).getLocal(); - assertThat(indices, hasItem(SecurityLifecycleService.SECURITY_INDEX_NAME)); + assertThat(indices, hasItem(SecurityIndexManager.SECURITY_INDEX_NAME)); } } @@ -1214,7 +1214,7 @@ public void testXPackUserDoesNotHaveAccessToSecurityIndex() { SearchRequest request = new SearchRequest(); final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(XPackUser.INSTANCE, SearchAction.NAME); List indices = resolveIndices(request, authorizedIndices).getLocal(); - assertThat(indices, not(hasItem(SecurityLifecycleService.SECURITY_INDEX_NAME))); + assertThat(indices, not(hasItem(SecurityIndexManager.SECURITY_INDEX_NAME))); } public void testNonXPackUserAccessingSecurityIndex() { @@ -1226,7 +1226,7 @@ public void testNonXPackUserAccessingSecurityIndex() { SearchRequest request = new SearchRequest(); final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(allAccessUser, SearchAction.NAME); List indices = resolveIndices(request, authorizedIndices).getLocal(); - assertThat(indices, not(hasItem(SecurityLifecycleService.SECURITY_INDEX_NAME))); + assertThat(indices, not(hasItem(SecurityIndexManager.SECURITY_INDEX_NAME))); } { @@ -1234,7 +1234,7 @@ public void testNonXPackUserAccessingSecurityIndex() { aliasesRequest.addAliasAction(AliasActions.add().alias("security_alias1").index("*")); final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(allAccessUser, IndicesAliasesAction.NAME); List indices = resolveIndices(aliasesRequest, authorizedIndices).getLocal(); - assertThat(indices, not(hasItem(SecurityLifecycleService.SECURITY_INDEX_NAME))); + assertThat(indices, not(hasItem(SecurityIndexManager.SECURITY_INDEX_NAME))); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java index ab6664b53b0fb..a2c70db3b63e8 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java @@ -41,8 +41,6 @@ import org.elasticsearch.xpack.core.security.action.role.PutRoleRequest; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges; -import org.elasticsearch.xpack.security.SecurityLifecycleService; -import org.elasticsearch.xpack.security.audit.index.IndexAuditTrail; import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.elasticsearch.xpack.security.test.SecurityTestUtils; import org.junit.After; @@ -58,7 +56,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE; -import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; @@ -189,10 +187,9 @@ public void testPutOfRoleWithFlsDlsUnlicensed() throws IOException { final ClusterService clusterService = mock(ClusterService.class); final XPackLicenseState licenseState = mock(XPackLicenseState.class); final AtomicBoolean methodCalled = new AtomicBoolean(false); - final SecurityLifecycleService securityLifecycleService = - new SecurityLifecycleService(Settings.EMPTY, clusterService, threadPool, client, - mock(IndexAuditTrail.class)); - final NativeRolesStore rolesStore = new NativeRolesStore(Settings.EMPTY, client, licenseState, securityLifecycleService) { + final SecurityIndexManager securityIndex = + new SecurityIndexManager(Settings.EMPTY, client, SecurityIndexManager.SECURITY_INDEX_NAME, clusterService); + final NativeRolesStore rolesStore = new NativeRolesStore(Settings.EMPTY, client, licenseState, securityIndex) { @Override void innerPutRole(final PutRoleRequest request, final RoleDescriptor role, final ActionListener listener) { if (methodCalled.compareAndSet(false, true)) { @@ -203,7 +200,7 @@ void innerPutRole(final PutRoleRequest request, final RoleDescriptor role, final } }; // setup the roles store so the security index exists - securityLifecycleService.clusterChanged(new ClusterChangedEvent( + securityIndex.clusterChanged(new ClusterChangedEvent( "fls_dls_license", getClusterStateWithSecurityIndex(), getEmptyClusterState())); PutRoleRequest putRoleRequest = new PutRoleRequest(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java index fe51f2beca34d..b5b67c7e7b2c1 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java @@ -37,6 +37,7 @@ import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -51,7 +52,7 @@ import org.junit.Before; import static org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE; -import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_TEMPLATE_NAME; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.TEMPLATE_VERSION_PATTERN; import static org.hamcrest.Matchers.equalTo; @@ -74,6 +75,7 @@ public void setUpManager() { when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY)); when(mockClient.threadPool()).thenReturn(threadPool); when(mockClient.settings()).thenReturn(Settings.EMPTY); + final ClusterService clusterService = mock(ClusterService.class); actions = new LinkedHashMap<>(); final Client client = new FilterClient(mockClient) { @@ -88,7 +90,7 @@ void doExecute(Action action, Request request actions.put(action, map); } }; - manager = new SecurityIndexManager(Settings.EMPTY, client, INDEX_NAME); + manager = new SecurityIndexManager(Settings.EMPTY, client, INDEX_NAME, clusterService); } public void testIndexWithUpToDateMappingAndTemplate() throws IOException { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java index 63c267eb816fc..aa4982cce3f84 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java @@ -40,7 +40,7 @@ import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; import static java.nio.file.StandardOpenOption.WRITE; import static org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE; -import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_INDEX_NAME; +import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; import static org.junit.Assert.assertEquals; public class SecurityTestUtils { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/XPackUserTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/XPackUserTests.java index 99c2ae635f6a6..e7b31d88eda19 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/XPackUserTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/user/XPackUserTests.java @@ -12,8 +12,8 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.security.index.IndexAuditTrailField; import org.elasticsearch.xpack.core.security.user.XPackUser; -import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.audit.index.IndexNameResolver; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.hamcrest.Matchers; import org.joda.time.DateTime; @@ -31,8 +31,8 @@ public void testXPackUserCanAccessNonSecurityIndices() { public void testXPackUserCannotAccessSecurityIndex() { final String action = randomFrom(GetAction.NAME, SearchAction.NAME, IndexAction.NAME); final Predicate predicate = XPackUser.ROLE.indices().allowedIndicesMatcher(action); - assertThat(predicate.test(SecurityLifecycleService.SECURITY_INDEX_NAME), Matchers.is(false)); - assertThat(predicate.test(SecurityLifecycleService.INTERNAL_SECURITY_INDEX), Matchers.is(false)); + assertThat(predicate.test(SecurityIndexManager.SECURITY_INDEX_NAME), Matchers.is(false)); + assertThat(predicate.test(SecurityIndexManager.INTERNAL_SECURITY_INDEX), Matchers.is(false)); } public void testXPackUserCanReadAuditTrail() { From 09329eb84f88c9fd241dcc9d33b3eb7478ee6037 Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Tue, 15 May 2018 22:46:46 +0300 Subject: [PATCH 02/34] SQL: Verify GROUP BY ordering on grouped columns (#30585) Due to the way composite aggregation works, ordering in GROUP BY can be applied only through grouped columns which now the analyzer verifier enforces. Fix 29900 --- .../xpack/sql/analysis/analyzer/Verifier.java | 23 ++++++++++++++++--- .../analyzer/VerifierErrorMessagesTests.java | 19 +++++++++++++-- .../planner/VerifierErrorMessagesTests.java | 14 +++++++++++ 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java index f5147b84468b7..6f8be61b463fd 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java @@ -211,12 +211,13 @@ static Collection verify(LogicalPlan plan) { /** * Check validity of Aggregate/GroupBy. - * This rule is needed for two reasons: + * This rule is needed for multiple reasons: * 1. a user might specify an invalid aggregate (SELECT foo GROUP BY bar) * 2. the order/having might contain a non-grouped attribute. This is typically * caught by the Analyzer however if wrapped in a function (ABS()) it gets resolved * (because the expression gets resolved little by little without being pushed down, * without the Analyzer modifying anything. + * 3. composite agg (used for GROUP BY) allows ordering only on the group keys */ private static boolean checkGroupBy(LogicalPlan p, Set localFailures, Map resolvedFunctions, Set groupingFailures) { @@ -225,7 +226,7 @@ && checkGroupByOrder(p, localFailures, groupingFailures, resolvedFunctions) && checkGroupByHaving(p, localFailures, groupingFailures, resolvedFunctions); } - // check whether an orderBy failed + // check whether an orderBy failed or if it occurs on a non-key private static boolean checkGroupByOrder(LogicalPlan p, Set localFailures, Set groupingFailures, Map functions) { if (p instanceof OrderBy) { @@ -234,7 +235,23 @@ private static boolean checkGroupByOrder(LogicalPlan p, Set localFailur Aggregate a = (Aggregate) o.child(); Map> missing = new LinkedHashMap<>(); - o.order().forEach(oe -> oe.collectFirstChildren(c -> checkGroupMatch(c, oe, a.groupings(), missing, functions))); + o.order().forEach(oe -> { + Expression e = oe.child(); + // cannot order by aggregates (not supported by composite) + if (Functions.isAggregate(e)) { + missing.put(e, oe); + return; + } + + // make sure to compare attributes directly + if (Expressions.anyMatch(a.groupings(), + g -> e.semanticEquals(e instanceof Attribute ? Expressions.attribute(g) : g))) { + return; + } + + // nothing matched, cannot group by it + missing.put(e, oe); + }); if (!missing.isEmpty()) { String plural = missing.size() > 1 ? "s" : StringUtils.EMPTY; diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java index 355c4d2f7b763..60875e0194a0c 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java @@ -111,7 +111,7 @@ public void testGroupByOrderByNonGrouped() { } public void testGroupByOrderByScalarOverNonGrouped() { - assertEquals("1:50: Cannot order by non-grouped column [date], expected [text]", + assertEquals("1:50: Cannot order by non-grouped column [YEAR(date [UTC])], expected [text]", verify("SELECT MAX(int) FROM test GROUP BY text ORDER BY YEAR(date)")); } @@ -144,4 +144,19 @@ public void testUnsupportedType() { assertEquals("1:8: Cannot use field [unsupported] type [ip_range] as is unsupported", verify("SELECT unsupported FROM test")); } -} + + public void testGroupByOrderByNonKey() { + assertEquals("1:52: Cannot order by non-grouped column [a], expected [bool]", + verify("SELECT AVG(int) a FROM test GROUP BY bool ORDER BY a")); + } + + public void testGroupByOrderByFunctionOverKey() { + assertEquals("1:44: Cannot order by non-grouped column [MAX(int)], expected [int]", + verify("SELECT int FROM test GROUP BY int ORDER BY MAX(int)")); + } + + public void testGroupByOrderByScore() { + assertEquals("1:44: Cannot order by non-grouped column [SCORE()], expected [int]", + verify("SELECT int FROM test GROUP BY int ORDER BY SCORE()")); + } +} \ No newline at end of file diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/VerifierErrorMessagesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/VerifierErrorMessagesTests.java index 154885261fdb8..5d6f479b7558d 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/VerifierErrorMessagesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/VerifierErrorMessagesTests.java @@ -49,4 +49,18 @@ public void testMultiGroupBy() { assertEquals("1:32: Currently, only a single expression can be used with GROUP BY; please select one of [bool, keyword]", verify("SELECT bool FROM test GROUP BY bool, keyword")); } + + // + // TODO potential improvements + // + // regarding resolution + // public void testGroupByOrderByKeyAlias() { + // assertEquals("1:8: Cannot use field [unsupported] type [ip_range] as is unsupported", + // verify("SELECT int i FROM test GROUP BY int ORDER BY i")); + // } + // + // public void testGroupByAlias() { + // assertEquals("1:8: Cannot use field [unsupported] type [ip_range] as is unsupported", + // verify("SELECT int i FROM test GROUP BY i ORDER BY int")); + // } } From 03dd2ab499742336263d09c9b69f62a8a76413da Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Tue, 15 May 2018 22:49:05 +0300 Subject: [PATCH 03/34] SQL: eliminate disabled tests --- .../sql/planner/VerifierErrorMessagesTests.java | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/VerifierErrorMessagesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/VerifierErrorMessagesTests.java index 5d6f479b7558d..154885261fdb8 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/VerifierErrorMessagesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/VerifierErrorMessagesTests.java @@ -49,18 +49,4 @@ public void testMultiGroupBy() { assertEquals("1:32: Currently, only a single expression can be used with GROUP BY; please select one of [bool, keyword]", verify("SELECT bool FROM test GROUP BY bool, keyword")); } - - // - // TODO potential improvements - // - // regarding resolution - // public void testGroupByOrderByKeyAlias() { - // assertEquals("1:8: Cannot use field [unsupported] type [ip_range] as is unsupported", - // verify("SELECT int i FROM test GROUP BY int ORDER BY i")); - // } - // - // public void testGroupByAlias() { - // assertEquals("1:8: Cannot use field [unsupported] type [ip_range] as is unsupported", - // verify("SELECT int i FROM test GROUP BY i ORDER BY int")); - // } } From 4f9dd3716910b1f896b72dc5e548b45d1ae5f18a Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Tue, 15 May 2018 13:07:58 -0700 Subject: [PATCH 04/34] Add support for search templates to the high-level REST client. (#30473) --- client/rest-high-level/build.gradle | 1 + .../client/RequestConverters.java | 33 ++- .../client/RestHighLevelClient.java | 28 +++ .../client/RequestConvertersTests.java | 125 ++++++++--- .../org/elasticsearch/client/SearchIT.java | 104 +++++++++ .../documentation/SearchDocumentationIT.java | 129 +++++++++++ .../search/search-template.asciidoc | 117 ++++++++++ .../high-level/supported-apis.asciidoc | 2 + .../RestMultiSearchTemplateAction.java | 2 +- .../RestRenderSearchTemplateAction.java | 2 +- .../mustache/RestSearchTemplateAction.java | 33 +-- .../mustache/SearchTemplateRequest.java | 85 ++++++- .../mustache/SearchTemplateResponse.java | 33 ++- .../script/mustache/SearchTemplateIT.java | 6 +- .../mustache/SearchTemplateRequestTests.java | 152 +++++-------- .../SearchTemplateRequestXContentTests.java | 197 ++++++++++++++++ .../mustache/SearchTemplateResponseTests.java | 211 ++++++++++++++++++ .../search/RandomSearchRequestGenerator.java | 2 +- 18 files changed, 1090 insertions(+), 172 deletions(-) create mode 100644 docs/java-rest/high-level/search/search-template.asciidoc create mode 100644 modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestXContentTests.java create mode 100644 modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateResponseTests.java diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle index c273e76a92aed..222de9608aeb9 100644 --- a/client/rest-high-level/build.gradle +++ b/client/rest-high-level/build.gradle @@ -40,6 +40,7 @@ dependencies { compile "org.elasticsearch.plugin:parent-join-client:${version}" compile "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}" compile "org.elasticsearch.plugin:rank-eval-client:${version}" + compile "org.elasticsearch.plugin:lang-mustache-client:${version}" testCompile "org.elasticsearch.client:test:${version}" testCompile "org.elasticsearch.test:framework:${version}" diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index 2e7b4ba74cc39..310aafcb6b817 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -80,6 +80,7 @@ import org.elasticsearch.index.VersionType; import org.elasticsearch.index.rankeval.RankEvalRequest; import org.elasticsearch.rest.action.search.RestSearchAction; +import org.elasticsearch.script.mustache.SearchTemplateRequest; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import java.io.ByteArrayOutputStream; @@ -458,6 +459,15 @@ static Request search(SearchRequest searchRequest) throws IOException { Request request = new Request(HttpPost.METHOD_NAME, endpoint(searchRequest.indices(), searchRequest.types(), "_search")); Params params = new Params(request); + addSearchRequestParams(params, searchRequest); + + if (searchRequest.source() != null) { + request.setEntity(createEntity(searchRequest.source(), REQUEST_BODY_CONTENT_TYPE)); + } + return request; + } + + private static void addSearchRequestParams(Params params, SearchRequest searchRequest) { params.putParam(RestSearchAction.TYPED_KEYS_PARAM, "true"); params.withRouting(searchRequest.routing()); params.withPreference(searchRequest.preference()); @@ -473,11 +483,6 @@ static Request search(SearchRequest searchRequest) throws IOException { if (searchRequest.scroll() != null) { params.putParam("scroll", searchRequest.scroll().keepAlive()); } - - if (searchRequest.source() != null) { - request.setEntity(createEntity(searchRequest.source(), REQUEST_BODY_CONTENT_TYPE)); - } - return request; } static Request searchScroll(SearchScrollRequest searchScrollRequest) throws IOException { @@ -507,6 +512,24 @@ static Request multiSearch(MultiSearchRequest multiSearchRequest) throws IOExcep return request; } + static Request searchTemplate(SearchTemplateRequest searchTemplateRequest) throws IOException { + Request request; + + if (searchTemplateRequest.isSimulate()) { + request = new Request(HttpGet.METHOD_NAME, "_render/template"); + } else { + SearchRequest searchRequest = searchTemplateRequest.getRequest(); + String endpoint = endpoint(searchRequest.indices(), searchRequest.types(), "_search/template"); + request = new Request(HttpGet.METHOD_NAME, endpoint); + + Params params = new Params(request); + addSearchRequestParams(params, searchRequest); + } + + request.setEntity(createEntity(searchTemplateRequest, REQUEST_BODY_CONTENT_TYPE)); + return request; + } + static Request existsAlias(GetAliasesRequest getAliasesRequest) { if ((getAliasesRequest.indices() == null || getAliasesRequest.indices().length == 0) && (getAliasesRequest.aliases() == null || getAliasesRequest.aliases().length == 0)) { diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index 1985d6bd06dd4..5dbf2709d9988 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -64,6 +64,8 @@ import org.elasticsearch.plugins.spi.NamedXContentProvider; import org.elasticsearch.rest.BytesRestResponse; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.script.mustache.SearchTemplateRequest; +import org.elasticsearch.script.mustache.SearchTemplateResponse; import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrixAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.adjacency.ParsedAdjacencyMatrix; @@ -501,6 +503,32 @@ public final void clearScrollAsync(ClearScrollRequest clearScrollRequest, listener, emptySet(), headers); } + /** + * Executes a request using the Search Template API. + * + * See Search Template API + * on elastic.co. + */ + public final SearchTemplateResponse searchTemplate(SearchTemplateRequest searchTemplateRequest, + Header... headers) throws IOException { + return performRequestAndParseEntity(searchTemplateRequest, RequestConverters::searchTemplate, + SearchTemplateResponse::fromXContent, emptySet(), headers); + } + + /** + * Asynchronously executes a request using the Search Template API + * + * See Search Template API + * on elastic.co. + */ + public final void searchTemplateAsync(SearchTemplateRequest searchTemplateRequest, + ActionListener listener, + Header... headers) { + performRequestAsyncAndParseEntity(searchTemplateRequest, RequestConverters::searchTemplate, + SearchTemplateResponse::fromXContent, listener, emptySet(), headers); + } + + /** * Executes a request using the Ranking Evaluation API. * diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index 2d4ef8b6413d9..9c75d67e04304 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -95,6 +95,8 @@ import org.elasticsearch.index.rankeval.RatedRequest; import org.elasticsearch.index.rankeval.RestRankEvalAction; import org.elasticsearch.rest.action.search.RestSearchAction; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.script.mustache.SearchTemplateRequest; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValueType; @@ -1011,36 +1013,7 @@ public void testSearch() throws Exception { searchRequest.types(types); Map expectedParams = new HashMap<>(); - expectedParams.put(RestSearchAction.TYPED_KEYS_PARAM, "true"); - if (randomBoolean()) { - searchRequest.routing(randomAlphaOfLengthBetween(3, 10)); - expectedParams.put("routing", searchRequest.routing()); - } - if (randomBoolean()) { - searchRequest.preference(randomAlphaOfLengthBetween(3, 10)); - expectedParams.put("preference", searchRequest.preference()); - } - if (randomBoolean()) { - searchRequest.searchType(randomFrom(SearchType.values())); - } - expectedParams.put("search_type", searchRequest.searchType().name().toLowerCase(Locale.ROOT)); - if (randomBoolean()) { - searchRequest.requestCache(randomBoolean()); - expectedParams.put("request_cache", Boolean.toString(searchRequest.requestCache())); - } - if (randomBoolean()) { - searchRequest.allowPartialSearchResults(randomBoolean()); - expectedParams.put("allow_partial_search_results", Boolean.toString(searchRequest.allowPartialSearchResults())); - } - if (randomBoolean()) { - searchRequest.setBatchedReduceSize(randomIntBetween(2, Integer.MAX_VALUE)); - } - expectedParams.put("batched_reduce_size", Integer.toString(searchRequest.getBatchedReduceSize())); - if (randomBoolean()) { - searchRequest.scroll(randomTimeValue()); - expectedParams.put("scroll", searchRequest.scroll().keepAlive().getStringRep()); - } - + setRandomSearchParams(searchRequest, expectedParams); setRandomIndicesOptions(searchRequest::indicesOptions, searchRequest::indicesOptions, expectedParams); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); @@ -1189,6 +1162,65 @@ public void testClearScroll() throws IOException { assertEquals(REQUEST_BODY_CONTENT_TYPE.mediaTypeWithoutParameters(), request.getEntity().getContentType().getValue()); } + public void testSearchTemplate() throws Exception { + // Create a random request. + String[] indices = randomIndicesNames(0, 5); + SearchRequest searchRequest = new SearchRequest(indices); + + Map expectedParams = new HashMap<>(); + setRandomSearchParams(searchRequest, expectedParams); + setRandomIndicesOptions(searchRequest::indicesOptions, searchRequest::indicesOptions, expectedParams); + + SearchTemplateRequest searchTemplateRequest = new SearchTemplateRequest(searchRequest); + + searchTemplateRequest.setScript("{\"query\": { \"match\" : { \"{{field}}\" : \"{{value}}\" }}}"); + searchTemplateRequest.setScriptType(ScriptType.INLINE); + searchTemplateRequest.setProfile(randomBoolean()); + + Map scriptParams = new HashMap<>(); + scriptParams.put("field", "name"); + scriptParams.put("value", "soren"); + searchTemplateRequest.setScriptParams(scriptParams); + + // Verify that the resulting REST request looks as expected. + Request request = RequestConverters.searchTemplate(searchTemplateRequest); + StringJoiner endpoint = new StringJoiner("/", "/", ""); + String index = String.join(",", indices); + if (Strings.hasLength(index)) { + endpoint.add(index); + } + endpoint.add("_search/template"); + + assertEquals(HttpGet.METHOD_NAME, request.getMethod()); + assertEquals(endpoint.toString(), request.getEndpoint()); + assertEquals(expectedParams, request.getParameters()); + assertToXContentBody(searchTemplateRequest, request.getEntity()); + } + + public void testRenderSearchTemplate() throws Exception { + // Create a simple request. + SearchTemplateRequest searchTemplateRequest = new SearchTemplateRequest(); + searchTemplateRequest.setSimulate(true); // Setting simulate true means the template should only be rendered. + + searchTemplateRequest.setScript("template1"); + searchTemplateRequest.setScriptType(ScriptType.STORED); + searchTemplateRequest.setProfile(randomBoolean()); + + Map scriptParams = new HashMap<>(); + scriptParams.put("field", "name"); + scriptParams.put("value", "soren"); + searchTemplateRequest.setScriptParams(scriptParams); + + // Verify that the resulting REST request looks as expected. + Request request = RequestConverters.searchTemplate(searchTemplateRequest); + String endpoint = "_render/template"; + + assertEquals(HttpGet.METHOD_NAME, request.getMethod()); + assertEquals(endpoint, request.getEndpoint()); + assertEquals(Collections.emptyMap(), request.getParameters()); + assertToXContentBody(searchTemplateRequest, request.getEntity()); + } + public void testExistsAlias() { GetAliasesRequest getAliasesRequest = new GetAliasesRequest(); String[] indices = randomBoolean() ? null : randomIndicesNames(0, 5); @@ -1662,6 +1694,39 @@ private static void randomizeFetchSourceContextParams(Consumer expectedParams) { + expectedParams.put(RestSearchAction.TYPED_KEYS_PARAM, "true"); + if (randomBoolean()) { + searchRequest.routing(randomAlphaOfLengthBetween(3, 10)); + expectedParams.put("routing", searchRequest.routing()); + } + if (randomBoolean()) { + searchRequest.preference(randomAlphaOfLengthBetween(3, 10)); + expectedParams.put("preference", searchRequest.preference()); + } + if (randomBoolean()) { + searchRequest.searchType(randomFrom(SearchType.values())); + } + expectedParams.put("search_type", searchRequest.searchType().name().toLowerCase(Locale.ROOT)); + if (randomBoolean()) { + searchRequest.requestCache(randomBoolean()); + expectedParams.put("request_cache", Boolean.toString(searchRequest.requestCache())); + } + if (randomBoolean()) { + searchRequest.allowPartialSearchResults(randomBoolean()); + expectedParams.put("allow_partial_search_results", Boolean.toString(searchRequest.allowPartialSearchResults())); + } + if (randomBoolean()) { + searchRequest.setBatchedReduceSize(randomIntBetween(2, Integer.MAX_VALUE)); + } + expectedParams.put("batched_reduce_size", Integer.toString(searchRequest.getBatchedReduceSize())); + if (randomBoolean()) { + searchRequest.scroll(randomTimeValue()); + expectedParams.put("scroll", searchRequest.scroll().keepAlive().getStringRep()); + } + } + private static void setRandomIndicesOptions(Consumer setter, Supplier getter, Map expectedParams) { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java index 549b4ce0a85c5..e147642fc73bd 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java @@ -38,8 +38,11 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.ScriptQueryBuilder; import org.elasticsearch.index.query.TermsQueryBuilder; @@ -48,6 +51,8 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; +import org.elasticsearch.script.mustache.SearchTemplateRequest; +import org.elasticsearch.script.mustache.SearchTemplateResponse; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.bucket.range.Range; @@ -69,10 +74,12 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; import static org.hamcrest.Matchers.both; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.either; @@ -733,6 +740,103 @@ public void testMultiSearch_failure() throws Exception { assertThat(multiSearchResponse.getResponses()[1].getResponse(), nullValue()); } + public void testSearchTemplate() throws IOException { + SearchTemplateRequest searchTemplateRequest = new SearchTemplateRequest(); + searchTemplateRequest.setRequest(new SearchRequest("index")); + + searchTemplateRequest.setScriptType(ScriptType.INLINE); + searchTemplateRequest.setScript( + "{" + + " \"query\": {" + + " \"match\": {" + + " \"num\": {{number}}" + + " }" + + " }" + + "}"); + + Map scriptParams = new HashMap<>(); + scriptParams.put("number", 10); + searchTemplateRequest.setScriptParams(scriptParams); + + searchTemplateRequest.setExplain(true); + searchTemplateRequest.setProfile(true); + + SearchTemplateResponse searchTemplateResponse = execute(searchTemplateRequest, + highLevelClient()::searchTemplate, + highLevelClient()::searchTemplateAsync); + + assertNull(searchTemplateResponse.getSource()); + + SearchResponse searchResponse = searchTemplateResponse.getResponse(); + assertNotNull(searchResponse); + + assertEquals(1, searchResponse.getHits().totalHits); + assertEquals(1, searchResponse.getHits().getHits().length); + assertThat(searchResponse.getHits().getMaxScore(), greaterThan(0f)); + + SearchHit hit = searchResponse.getHits().getHits()[0]; + assertNotNull(hit.getExplanation()); + + assertFalse(searchResponse.getProfileResults().isEmpty()); + } + + public void testNonExistentSearchTemplate() { + SearchTemplateRequest searchTemplateRequest = new SearchTemplateRequest(); + searchTemplateRequest.setRequest(new SearchRequest("index")); + + searchTemplateRequest.setScriptType(ScriptType.STORED); + searchTemplateRequest.setScript("non-existent"); + searchTemplateRequest.setScriptParams(Collections.emptyMap()); + + ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, + () -> execute(searchTemplateRequest, + highLevelClient()::searchTemplate, + highLevelClient()::searchTemplateAsync)); + + assertEquals(RestStatus.NOT_FOUND, exception.status()); + } + + public void testRenderSearchTemplate() throws IOException { + SearchTemplateRequest searchTemplateRequest = new SearchTemplateRequest(); + + searchTemplateRequest.setScriptType(ScriptType.INLINE); + searchTemplateRequest.setScript( + "{" + + " \"query\": {" + + " \"match\": {" + + " \"num\": {{number}}" + + " }" + + " }" + + "}"); + + Map scriptParams = new HashMap<>(); + scriptParams.put("number", 10); + searchTemplateRequest.setScriptParams(scriptParams); + + // Setting simulate true causes the template to only be rendered. + searchTemplateRequest.setSimulate(true); + + SearchTemplateResponse searchTemplateResponse = execute(searchTemplateRequest, + highLevelClient()::searchTemplate, + highLevelClient()::searchTemplateAsync); + assertNull(searchTemplateResponse.getResponse()); + + BytesReference expectedSource = BytesReference.bytes( + XContentFactory.jsonBuilder() + .startObject() + .startObject("query") + .startObject("match") + .field("num", 10) + .endObject() + .endObject() + .endObject()); + + BytesReference actualSource = searchTemplateResponse.getSource(); + assertNotNull(actualSource); + + assertToXContentEquivalent(expectedSource, actualSource, XContentType.JSON); + } + public void testFieldCaps() throws IOException { FieldCapabilitiesRequest request = new FieldCapabilitiesRequest() .indices("index1", "index2") diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java index 8a12016025c3e..463c5f7d12f5e 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java @@ -41,7 +41,11 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.ESRestHighLevelClientTestCase; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.unit.TimeValue; @@ -60,6 +64,9 @@ import org.elasticsearch.index.rankeval.RatedRequest; import org.elasticsearch.index.rankeval.RatedSearchHit; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.script.mustache.SearchTemplateRequest; +import org.elasticsearch.script.mustache.SearchTemplateResponse; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; @@ -92,6 +99,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; @@ -706,9 +714,130 @@ public void onFailure(Exception e) { } } + public void testSearchTemplateWithInlineScript() throws Exception { + indexSearchTestData(); + RestHighLevelClient client = highLevelClient(); + + // tag::search-template-request-inline + SearchTemplateRequest request = new SearchTemplateRequest(); + request.setRequest(new SearchRequest("posts")); // <1> + + request.setScriptType(ScriptType.INLINE); + request.setScript( // <2> + "{" + + " \"query\": { \"match\" : { \"{{field}}\" : \"{{value}}\" } }," + + " \"size\" : \"{{size}}\"" + + "}"); + + Map scriptParams = new HashMap<>(); + scriptParams.put("field", "title"); + scriptParams.put("value", "elasticsearch"); + scriptParams.put("size", 5); + request.setScriptParams(scriptParams); // <3> + // end::search-template-request-inline + + // tag::search-template-response + SearchTemplateResponse response = client.searchTemplate(request); + SearchResponse searchResponse = response.getResponse(); + // end::search-template-response + + assertNotNull(searchResponse); + assertTrue(searchResponse.getHits().totalHits > 0); + + // tag::render-search-template-request + request.setSimulate(true); // <1> + // end::render-search-template-request + + // tag::render-search-template-response + SearchTemplateResponse renderResponse = client.searchTemplate(request); + BytesReference source = renderResponse.getSource(); // <1> + // end::render-search-template-response + + assertNotNull(source); + assertEquals(( + "{" + + " \"size\" : \"5\"," + + " \"query\": { \"match\" : { \"title\" : \"elasticsearch\" } }" + + "}").replaceAll("\\s+", ""), source.utf8ToString()); + } + + public void testSearchTemplateWithStoredScript() throws Exception { + indexSearchTestData(); + RestHighLevelClient client = highLevelClient(); + RestClient restClient = client(); + + // tag::register-script + Request scriptRequest = new Request("POST", "_scripts/title_search"); + scriptRequest.setJsonEntity( + "{" + + " \"script\": {" + + " \"lang\": \"mustache\"," + + " \"source\": {" + + " \"query\": { \"match\" : { \"{{field}}\" : \"{{value}}\" } }," + + " \"size\" : \"{{size}}\"" + + " }" + + " }" + + "}"); + Response scriptResponse = restClient.performRequest(scriptRequest); + // end::register-script + assertEquals(RestStatus.OK.getStatus(), scriptResponse.getStatusLine().getStatusCode()); + + // tag::search-template-request-stored + SearchTemplateRequest request = new SearchTemplateRequest(); + request.setRequest(new SearchRequest("posts")); + + request.setScriptType(ScriptType.STORED); + request.setScript("title_search"); + + Map params = new HashMap<>(); + params.put("field", "title"); + params.put("value", "elasticsearch"); + params.put("size", 5); + request.setScriptParams(params); + // end::search-template-request-stored + + // tag::search-template-request-options + request.setExplain(true); + request.setProfile(true); + // end::search-template-request-options + + // tag::search-template-execute + SearchTemplateResponse response = client.searchTemplate(request); + // end::search-template-execute + + SearchResponse searchResponse = response.getResponse(); + assertNotNull(searchResponse); + assertTrue(searchResponse.getHits().totalHits > 0); + + // tag::search-template-execute-listener + ActionListener listener = new ActionListener() { + @Override + public void onResponse(SearchTemplateResponse response) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::search-template-execute-listener + + // Replace the empty listener by a blocking listener for tests. + CountDownLatch latch = new CountDownLatch(1); + listener = new LatchedActionListener<>(listener, latch); + + // tag::search-template-execute-async + client.searchTemplateAsync(request, listener); // <1> + // end::search-template-execute-async + + assertTrue(latch.await(30L, TimeUnit.SECONDS)); + } + public void testFieldCaps() throws Exception { indexSearchTestData(); RestHighLevelClient client = highLevelClient(); + // tag::field-caps-request FieldCapabilitiesRequest request = new FieldCapabilitiesRequest() .fields("user") diff --git a/docs/java-rest/high-level/search/search-template.asciidoc b/docs/java-rest/high-level/search/search-template.asciidoc new file mode 100644 index 0000000000000..3f0dfb8ab28e0 --- /dev/null +++ b/docs/java-rest/high-level/search/search-template.asciidoc @@ -0,0 +1,117 @@ +[[java-rest-high-search-template]] +=== Search Template API + +The search template API allows for searches to be executed from a template based +on the mustache language, and also for previewing rendered templates. + +[[java-rest-high-search-template-request]] +==== Search Template Request + +===== Inline Templates + +In the most basic form of request, the search template is specified inline: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[search-template-request-inline] +-------------------------------------------------- +<1> The search is executed against the `posts` index. +<2> The template defines the structure of the search source. It is passed +as a string because mustache templates are not always valid JSON. +<3> Before running the search, the template is rendered with the provided parameters. + +===== Registered Templates + +Search templates can be registered in advance through stored scripts API. Note that +the stored scripts API is not yet available in the high-level REST client, so in this +example we use the low-level REST client. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[register-script] +-------------------------------------------------- + +Instead of providing an inline script, we can refer to this registered template in the request: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[search-template-request-stored] +-------------------------------------------------- + +===== Rendering Templates + +Given parameter values, a template can be rendered without executing a search: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[render-search-template-request] +-------------------------------------------------- +<1> Setting `simulate` to `true` causes the search template to only be rendered. + +Both inline and pre-registered templates can be rendered. + +===== Optional Arguments + +As in standard search requests, the `explain` and `profile` options are supported: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[search-template-request-options] +-------------------------------------------------- + +===== Additional References + +The {ref}/search-template.html[Search Template documentation] contains further examples of how search requests can be templated. + +[[java-rest-high-search-template-sync]] +==== Synchronous Execution + +The `searchTemplate` method executes the request synchronously: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[search-template-execute] +-------------------------------------------------- + +==== Asynchronous Execution + +A search template request can be executed asynchronously through the `searchTemplateAsync` +method: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[search-template-execute-async] +-------------------------------------------------- +<1> The `SearchTemplateRequest` to execute and the `ActionListener` to call when the execution completes. + +The asynchronous method does not block and returns immediately. Once the request completes, the +`ActionListener` is called back using the `onResponse` method if the execution completed successfully, +or using the `onFailure` method if it failed. + +A typical listener for `SearchTemplateResponse` is constructed as follows: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[search-template-execute-listener] +-------------------------------------------------- +<1> Called when the execution is successfully completed. +<2> Called when the whole `SearchTemplateRequest` fails. + +==== Search Template Response + +For a standard search template request, the response contains a `SearchResponse` object +with the result of executing the search: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[search-template-response] +-------------------------------------------------- + +If `simulate` was set to `true` in the request, then the response +will contain the rendered search source instead of a `SearchResponse`: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[render-search-template-response] +-------------------------------------------------- +<1> The rendered source in bytes, in our example `{"query": { "match" : { "title" : "elasticsearch" }}, "size" : 5}`. diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc index 2dee4643e73eb..62e65ec650bca 100644 --- a/docs/java-rest/high-level/supported-apis.asciidoc +++ b/docs/java-rest/high-level/supported-apis.asciidoc @@ -31,6 +31,7 @@ The Java High Level REST Client supports the following Search APIs: * <> * <> * <> +* <> * <> * <> * <> @@ -38,6 +39,7 @@ The Java High Level REST Client supports the following Search APIs: include::search/search.asciidoc[] include::search/scroll.asciidoc[] include::search/multi-search.asciidoc[] +include::search/search-template.asciidoc[] include::search/field-caps.asciidoc[] include::search/rank-eval.asciidoc[] diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java index fd797c4340a8f..9969e6b38e54a 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java @@ -77,7 +77,7 @@ public static MultiSearchTemplateRequest parseRequest(RestRequest restRequest, b RestMultiSearchAction.parseMultiLineRequest(restRequest, multiRequest.indicesOptions(), allowExplicitIndex, (searchRequest, bytes) -> { - SearchTemplateRequest searchTemplateRequest = RestSearchTemplateAction.parse(bytes); + SearchTemplateRequest searchTemplateRequest = SearchTemplateRequest.fromXContent(bytes); if (searchTemplateRequest.getScript() != null) { searchTemplateRequest.setRequest(searchRequest); multiRequest.add(searchTemplateRequest); diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestRenderSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestRenderSearchTemplateAction.java index d8c67839cb80f..75acc09424359 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestRenderSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestRenderSearchTemplateAction.java @@ -52,7 +52,7 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client // Creates the render template request SearchTemplateRequest renderRequest; try (XContentParser parser = request.contentOrSourceParamParser()) { - renderRequest = RestSearchTemplateAction.parse(parser); + renderRequest = SearchTemplateRequest.fromXContent(parser); } renderRequest.setSimulate(true); diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java index 7ab9aa6003334..f42afcc19b80f 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java @@ -47,33 +47,6 @@ public class RestSearchTemplateAction extends BaseRestHandler { private static final Set RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TYPED_KEYS_PARAM); - private static final ObjectParser PARSER; - static { - PARSER = new ObjectParser<>("search_template"); - PARSER.declareField((parser, request, s) -> - request.setScriptParams(parser.map()) - , new ParseField("params"), ObjectParser.ValueType.OBJECT); - PARSER.declareString((request, s) -> { - request.setScriptType(ScriptType.STORED); - request.setScript(s); - }, new ParseField("id")); - PARSER.declareBoolean(SearchTemplateRequest::setExplain, new ParseField("explain")); - PARSER.declareBoolean(SearchTemplateRequest::setProfile, new ParseField("profile")); - PARSER.declareField((parser, request, value) -> { - request.setScriptType(ScriptType.INLINE); - if (parser.currentToken() == XContentParser.Token.START_OBJECT) { - //convert the template to json which is the only supported XContentType (see CustomMustacheFactory#createEncoder) - try (XContentBuilder builder = XContentFactory.jsonBuilder()) { - request.setScript(Strings.toString(builder.copyCurrentStructure(parser))); - } catch (IOException e) { - throw new ParsingException(parser.getTokenLocation(), "Could not parse inline template", e); - } - } else { - request.setScript(parser.text()); - } - }, new ParseField("source", "inline", "template"), ObjectParser.ValueType.OBJECT_OR_STRING); - } - public RestSearchTemplateAction(Settings settings, RestController controller) { super(settings); @@ -99,17 +72,13 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client // Creates the search template request SearchTemplateRequest searchTemplateRequest; try (XContentParser parser = request.contentOrSourceParamParser()) { - searchTemplateRequest = PARSER.parse(parser, new SearchTemplateRequest(), null); + searchTemplateRequest = SearchTemplateRequest.fromXContent(parser); } searchTemplateRequest.setRequest(searchRequest); return channel -> client.execute(SearchTemplateAction.INSTANCE, searchTemplateRequest, new RestStatusToXContentListener<>(channel)); } - public static SearchTemplateRequest parse(XContentParser parser) throws IOException { - return PARSER.parse(parser, new SearchTemplateRequest(), null); - } - @Override protected Set responseParams() { return RESPONSE_PARAMS; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequest.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequest.java index b0186b7b0e3cf..da3cc3688149c 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequest.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequest.java @@ -23,19 +23,28 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.CompositeIndicesRequest; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.script.ScriptType; import java.io.IOException; import java.util.Map; +import java.util.Objects; import static org.elasticsearch.action.ValidateActions.addValidationError; /** * A request to execute a search based on a search template. */ -public class SearchTemplateRequest extends ActionRequest implements CompositeIndicesRequest { +public class SearchTemplateRequest extends ActionRequest implements CompositeIndicesRequest, ToXContentObject { private SearchRequest request; private boolean simulate = false; @@ -60,6 +69,24 @@ public SearchRequest getRequest() { return request; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SearchTemplateRequest request1 = (SearchTemplateRequest) o; + return simulate == request1.simulate && + explain == request1.explain && + profile == request1.profile && + Objects.equals(request, request1.request) && + scriptType == request1.scriptType && + Objects.equals(script, request1.script) && + Objects.equals(scriptParams, request1.scriptParams); + } + + @Override + public int hashCode() { + return Objects.hash(request, simulate, explain, profile, scriptType, script, scriptParams); + } public boolean isSimulate() { return simulate; @@ -134,6 +161,62 @@ public ActionRequestValidationException validate() { return validationException; } + private static ParseField ID_FIELD = new ParseField("id"); + private static ParseField SOURCE_FIELD = new ParseField("source", "inline", "template"); + + private static ParseField PARAMS_FIELD = new ParseField("params"); + private static ParseField EXPLAIN_FIELD = new ParseField("explain"); + private static ParseField PROFILE_FIELD = new ParseField("profile"); + + private static final ObjectParser PARSER; + static { + PARSER = new ObjectParser<>("search_template"); + PARSER.declareField((parser, request, s) -> + request.setScriptParams(parser.map()) + , PARAMS_FIELD, ObjectParser.ValueType.OBJECT); + PARSER.declareString((request, s) -> { + request.setScriptType(ScriptType.STORED); + request.setScript(s); + }, ID_FIELD); + PARSER.declareBoolean(SearchTemplateRequest::setExplain, EXPLAIN_FIELD); + PARSER.declareBoolean(SearchTemplateRequest::setProfile, PROFILE_FIELD); + PARSER.declareField((parser, request, value) -> { + request.setScriptType(ScriptType.INLINE); + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + //convert the template to json which is the only supported XContentType (see CustomMustacheFactory#createEncoder) + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + request.setScript(Strings.toString(builder.copyCurrentStructure(parser))); + } catch (IOException e) { + throw new ParsingException(parser.getTokenLocation(), "Could not parse inline template", e); + } + } else { + request.setScript(parser.text()); + } + }, SOURCE_FIELD, ObjectParser.ValueType.OBJECT_OR_STRING); + } + + public static SearchTemplateRequest fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, new SearchTemplateRequest(), null); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + + if (scriptType == ScriptType.STORED) { + builder.field(ID_FIELD.getPreferredName(), script); + } else if (scriptType == ScriptType.INLINE) { + builder.field(SOURCE_FIELD.getPreferredName(), script); + } else { + throw new UnsupportedOperationException("Unrecognized script type [" + scriptType + "]."); + } + + return builder.field(PARAMS_FIELD.getPreferredName(), scriptParams) + .field(EXPLAIN_FIELD.getPreferredName(), explain) + .field(PROFILE_FIELD.getPreferredName(), profile) + .endObject(); + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java index 792d993915992..500a5a399ef4a 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java @@ -21,18 +21,23 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.StatusToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestStatus; import java.io.IOException; import java.io.InputStream; +import java.util.Map; -public class SearchTemplateResponse extends ActionResponse implements StatusToXContentObject { +public class SearchTemplateResponse extends ActionResponse implements StatusToXContentObject { + public static ParseField TEMPLATE_OUTPUT_FIELD = new ParseField("template_output"); /** Contains the source of the rendered template **/ private BytesReference source; @@ -77,6 +82,30 @@ public void readFrom(StreamInput in) throws IOException { response = in.readOptionalStreamable(SearchResponse::new); } + public static SearchTemplateResponse fromXContent(XContentParser parser) throws IOException { + SearchTemplateResponse searchTemplateResponse = new SearchTemplateResponse(); + Map contentAsMap = parser.map(); + + if (contentAsMap.containsKey(TEMPLATE_OUTPUT_FIELD.getPreferredName())) { + Object source = contentAsMap.get(TEMPLATE_OUTPUT_FIELD.getPreferredName()); + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON) + .value(source); + searchTemplateResponse.setSource(BytesReference.bytes(builder)); + } else { + XContentType contentType = parser.contentType(); + XContentBuilder builder = XContentFactory.contentBuilder(contentType) + .map(contentAsMap); + XContentParser searchResponseParser = contentType.xContent().createParser( + parser.getXContentRegistry(), + parser.getDeprecationHandler(), + BytesReference.bytes(builder).streamInput()); + + SearchResponse searchResponse = SearchResponse.fromXContent(searchResponseParser); + searchTemplateResponse.setResponse(searchResponse); + } + return searchTemplateResponse; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { if (hasResponse()) { @@ -85,7 +114,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); //we can assume the template is always json as we convert it before compiling it try (InputStream stream = source.streamInput()) { - builder.rawField("template_output", stream, XContentType.JSON); + builder.rawField(TEMPLATE_OUTPUT_FIELD.getPreferredName(), stream, XContentType.JSON); } builder.endObject(); } diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java index 1529b655a5042..fe2fedf62b559 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java @@ -101,7 +101,7 @@ public void testTemplateQueryAsEscapedString() throws Exception { + " \"size\": 1" + " }" + "}"; - SearchTemplateRequest request = RestSearchTemplateAction.parse(createParser(JsonXContent.jsonXContent, query)); + SearchTemplateRequest request = SearchTemplateRequest.fromXContent(createParser(JsonXContent.jsonXContent, query)); request.setRequest(searchRequest); SearchTemplateResponse searchResponse = client().execute(SearchTemplateAction.INSTANCE, request).get(); assertThat(searchResponse.getResponse().getHits().getHits().length, equalTo(1)); @@ -122,7 +122,7 @@ public void testTemplateQueryAsEscapedStringStartingWithConditionalClause() thro + " \"use_size\": true" + " }" + "}"; - SearchTemplateRequest request = RestSearchTemplateAction.parse(createParser(JsonXContent.jsonXContent, templateString)); + SearchTemplateRequest request = SearchTemplateRequest.fromXContent(createParser(JsonXContent.jsonXContent, templateString)); request.setRequest(searchRequest); SearchTemplateResponse searchResponse = client().execute(SearchTemplateAction.INSTANCE, request).get(); assertThat(searchResponse.getResponse().getHits().getHits().length, equalTo(1)); @@ -143,7 +143,7 @@ public void testTemplateQueryAsEscapedStringWithConditionalClauseAtEnd() throws + " \"use_size\": true" + " }" + "}"; - SearchTemplateRequest request = RestSearchTemplateAction.parse(createParser(JsonXContent.jsonXContent, templateString)); + SearchTemplateRequest request = SearchTemplateRequest.fromXContent(createParser(JsonXContent.jsonXContent, templateString)); request.setRequest(searchRequest); SearchTemplateResponse searchResponse = client().execute(SearchTemplateAction.INSTANCE, request).get(); assertThat(searchResponse.getResponse().getHits().getHits().length, equalTo(1)); diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestTests.java index 9cdca70f0e1a6..7d4a6479727e2 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestTests.java @@ -19,117 +19,77 @@ package org.elasticsearch.script.mustache; -import org.elasticsearch.common.xcontent.XContentParseException; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.script.ScriptType; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.search.RandomSearchRequestGenerator; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.test.AbstractStreamableTestCase; import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.function.Consumer; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasEntry; -import static org.hamcrest.Matchers.hasItems; -import static org.hamcrest.Matchers.hasKey; -import static org.hamcrest.Matchers.nullValue; - -public class SearchTemplateRequestTests extends ESTestCase { - - public void testParseInlineTemplate() throws Exception { - String source = "{" + - " 'source' : {\n" + - " 'query': {\n" + - " 'terms': {\n" + - " 'status': [\n" + - " '{{#status}}',\n" + - " '{{.}}',\n" + - " '{{/status}}'\n" + - " ]\n" + - " }\n" + - " }\n" + - " }" + - "}"; - - SearchTemplateRequest request = RestSearchTemplateAction.parse(newParser(source)); - assertThat(request.getScript(), equalTo("{\"query\":{\"terms\":{\"status\":[\"{{#status}}\",\"{{.}}\",\"{{/status}}\"]}}}")); - assertThat(request.getScriptType(), equalTo(ScriptType.INLINE)); - assertThat(request.getScriptParams(), nullValue()); - } +public class SearchTemplateRequestTests extends AbstractStreamableTestCase { - public void testParseInlineTemplateWithParams() throws Exception { - String source = "{" + - " 'source' : {" + - " 'query': { 'match' : { '{{my_field}}' : '{{my_value}}' } }," + - " 'size' : '{{my_size}}'" + - " }," + - " 'params' : {" + - " 'my_field' : 'foo'," + - " 'my_value' : 'bar'," + - " 'my_size' : 5" + - " }" + - "}"; - - SearchTemplateRequest request = RestSearchTemplateAction.parse(newParser(source)); - assertThat(request.getScript(), equalTo("{\"query\":{\"match\":{\"{{my_field}}\":\"{{my_value}}\"}},\"size\":\"{{my_size}}\"}")); - assertThat(request.getScriptType(), equalTo(ScriptType.INLINE)); - assertThat(request.getScriptParams().size(), equalTo(3)); - assertThat(request.getScriptParams(), hasEntry("my_field", "foo")); - assertThat(request.getScriptParams(), hasEntry("my_value", "bar")); - assertThat(request.getScriptParams(), hasEntry("my_size", 5)); + @Override + protected SearchTemplateRequest createBlankInstance() { + return new SearchTemplateRequest(); } - public void testParseInlineTemplateAsString() throws Exception { - String source = "{'source' : '{\\\"query\\\":{\\\"bool\\\":{\\\"must\\\":{\\\"match\\\":{\\\"foo\\\":\\\"{{text}}\\\"}}}}}'}"; - - SearchTemplateRequest request = RestSearchTemplateAction.parse(newParser(source)); - assertThat(request.getScript(), equalTo("{\"query\":{\"bool\":{\"must\":{\"match\":{\"foo\":\"{{text}}\"}}}}}")); - assertThat(request.getScriptType(), equalTo(ScriptType.INLINE)); - assertThat(request.getScriptParams(), nullValue()); + @Override + protected SearchTemplateRequest createTestInstance() { + return createRandomRequest(); } - @SuppressWarnings("unchecked") - public void testParseInlineTemplateAsStringWithParams() throws Exception { - String source = "{'source' : '{\\\"query\\\":{\\\"match\\\":{\\\"{{field}}\\\":\\\"{{value}}\\\"}}}', " + - "'params': {'status': ['pending', 'published']}}"; - - SearchTemplateRequest request = RestSearchTemplateAction.parse(newParser(source)); - assertThat(request.getScript(), equalTo("{\"query\":{\"match\":{\"{{field}}\":\"{{value}}\"}}}")); - assertThat(request.getScriptType(), equalTo(ScriptType.INLINE)); - assertThat(request.getScriptParams().size(), equalTo(1)); - assertThat(request.getScriptParams(), hasKey("status")); - assertThat((List) request.getScriptParams().get("status"), hasItems("pending", "published")); + @Override + protected SearchTemplateRequest mutateInstance(SearchTemplateRequest instance) throws IOException { + List> mutators = new ArrayList<>(); + + mutators.add(request -> request.setScriptType( + randomValueOtherThan(request.getScriptType(), () -> randomFrom(ScriptType.values())))); + mutators.add(request -> request.setScript( + randomValueOtherThan(request.getScript(), () -> randomAlphaOfLength(50)))); + + mutators.add(request -> { + Map mutatedScriptParams = new HashMap<>(request.getScriptParams()); + String newField = randomValueOtherThanMany(mutatedScriptParams::containsKey, () -> randomAlphaOfLength(5)); + mutatedScriptParams.put(newField, randomAlphaOfLength(10)); + request.setScriptParams(mutatedScriptParams); + }); + + mutators.add(request -> request.setProfile(!request.isProfile())); + mutators.add(request -> request.setExplain(!request.isExplain())); + mutators.add(request -> request.setSimulate(!request.isSimulate())); + + mutators.add(request -> request.setRequest( + RandomSearchRequestGenerator.randomSearchRequest(SearchSourceBuilder::searchSource))); + + SearchTemplateRequest mutatedInstance = copyInstance(instance); + Consumer mutator = randomFrom(mutators); + mutator.accept(mutatedInstance); + return mutatedInstance; } - public void testParseStoredTemplate() throws Exception { - String source = "{'id' : 'storedTemplate'}"; - - SearchTemplateRequest request = RestSearchTemplateAction.parse(newParser(source)); - assertThat(request.getScript(), equalTo("storedTemplate")); - assertThat(request.getScriptType(), equalTo(ScriptType.STORED)); - assertThat(request.getScriptParams(), nullValue()); - } - public void testParseStoredTemplateWithParams() throws Exception { - String source = "{'id' : 'another_template', 'params' : {'bar': 'foo'}}"; + public static SearchTemplateRequest createRandomRequest() { + SearchTemplateRequest request = new SearchTemplateRequest(); + request.setScriptType(randomFrom(ScriptType.values())); + request.setScript(randomAlphaOfLength(50)); - SearchTemplateRequest request = RestSearchTemplateAction.parse(newParser(source)); - assertThat(request.getScript(), equalTo("another_template")); - assertThat(request.getScriptType(), equalTo(ScriptType.STORED)); - assertThat(request.getScriptParams().size(), equalTo(1)); - assertThat(request.getScriptParams(), hasEntry("bar", "foo")); - } + Map scriptParams = new HashMap<>(); + for (int i = 0; i < randomInt(10); i++) { + scriptParams.put(randomAlphaOfLength(5), randomAlphaOfLength(10)); + } + request.setScriptParams(scriptParams); - public void testParseWrongTemplate() { - // Unclosed template id - expectThrows(XContentParseException.class, () -> RestSearchTemplateAction.parse(newParser("{'id' : 'another_temp }"))); - } + request.setExplain(randomBoolean()); + request.setProfile(randomBoolean()); + request.setSimulate(randomBoolean()); - /** - * Creates a {@link XContentParser} with the given String while replacing single quote to double quotes. - */ - private XContentParser newParser(String s) throws IOException { - assertNotNull(s); - return createParser(JsonXContent.jsonXContent, s.replace("'", "\"")); + request.setRequest(RandomSearchRequestGenerator.randomSearchRequest( + SearchSourceBuilder::searchSource)); + return request; } } diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestXContentTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestXContentTests.java new file mode 100644 index 0000000000000..0e9e8ca628975 --- /dev/null +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateRequestXContentTests.java @@ -0,0 +1,197 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.mustache; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParseException; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.nullValue; + +public class SearchTemplateRequestXContentTests extends AbstractXContentTestCase { + + @Override + public SearchTemplateRequest createTestInstance() { + return SearchTemplateRequestTests.createRandomRequest(); + } + + @Override + protected SearchTemplateRequest doParseInstance(XContentParser parser) throws IOException { + return SearchTemplateRequest.fromXContent(parser); + } + + /** + * Note that when checking equality for xContent parsing, we omit two parts of the request: + * - The 'simulate' option, since this parameter is not included in the + * request's xContent (it's instead used to determine the request endpoint). + * - The random SearchRequest, since this component only affects the request + * parameters and also isn't captured in the request's xContent. + */ + @Override + protected void assertEqualInstances(SearchTemplateRequest expectedInstance, SearchTemplateRequest newInstance) { + assertTrue( + expectedInstance.isExplain() == newInstance.isExplain() && + expectedInstance.isProfile() == newInstance.isProfile() && + expectedInstance.getScriptType() == newInstance.getScriptType() && + Objects.equals(expectedInstance.getScript(), newInstance.getScript()) && + Objects.equals(expectedInstance.getScriptParams(), newInstance.getScriptParams())); + } + + @Override + protected boolean supportsUnknownFields() { + return false; + } + + public void testToXContentWithInlineTemplate() throws IOException { + SearchTemplateRequest request = new SearchTemplateRequest(); + + request.setScriptType(ScriptType.INLINE); + request.setScript("{\"query\": { \"match\" : { \"{{my_field}}\" : \"{{my_value}}\" } } }"); + request.setProfile(true); + + Map scriptParams = new HashMap<>(); + scriptParams.put("my_field", "foo"); + scriptParams.put("my_value", "bar"); + request.setScriptParams(scriptParams); + + XContentType contentType = randomFrom(XContentType.values()); + XContentBuilder expectedRequest = XContentFactory.contentBuilder(contentType) + .startObject() + .field("source", "{\"query\": { \"match\" : { \"{{my_field}}\" : \"{{my_value}}\" } } }") + .startObject("params") + .field("my_field", "foo") + .field("my_value", "bar") + .endObject() + .field("explain", false) + .field("profile", true) + .endObject(); + + XContentBuilder actualRequest = XContentFactory.contentBuilder(contentType); + request.toXContent(actualRequest, ToXContent.EMPTY_PARAMS); + + assertToXContentEquivalent(BytesReference.bytes(expectedRequest), + BytesReference.bytes(actualRequest), + contentType); + } + + public void testToXContentWithStoredTemplate() throws IOException { + SearchTemplateRequest request = new SearchTemplateRequest(); + + request.setScriptType(ScriptType.STORED); + request.setScript("match_template"); + request.setExplain(true); + + Map params = new HashMap<>(); + params.put("my_field", "foo"); + params.put("my_value", "bar"); + request.setScriptParams(params); + + XContentType contentType = randomFrom(XContentType.values()); + XContentBuilder expectedRequest = XContentFactory.contentBuilder(contentType) + .startObject() + .field("id", "match_template") + .startObject("params") + .field("my_field", "foo") + .field("my_value", "bar") + .endObject() + .field("explain", true) + .field("profile", false) + .endObject(); + + XContentBuilder actualRequest = XContentFactory.contentBuilder(contentType); + request.toXContent(actualRequest, ToXContent.EMPTY_PARAMS); + + assertToXContentEquivalent( + BytesReference.bytes(expectedRequest), + BytesReference.bytes(actualRequest), + contentType); + } + + public void testFromXContentWithEmbeddedTemplate() throws Exception { + String source = "{" + + " 'source' : {\n" + + " 'query': {\n" + + " 'terms': {\n" + + " 'status': [\n" + + " '{{#status}}',\n" + + " '{{.}}',\n" + + " '{{/status}}'\n" + + " ]\n" + + " }\n" + + " }\n" + + " }" + + "}"; + + SearchTemplateRequest request = SearchTemplateRequest.fromXContent(newParser(source)); + assertThat(request.getScript(), equalTo("{\"query\":{\"terms\":{\"status\":[\"{{#status}}\",\"{{.}}\",\"{{/status}}\"]}}}")); + assertThat(request.getScriptType(), equalTo(ScriptType.INLINE)); + assertThat(request.getScriptParams(), nullValue()); + } + + public void testFromXContentWithEmbeddedTemplateAndParams() throws Exception { + String source = "{" + + " 'source' : {" + + " 'query': { 'match' : { '{{my_field}}' : '{{my_value}}' } }," + + " 'size' : '{{my_size}}'" + + " }," + + " 'params' : {" + + " 'my_field' : 'foo'," + + " 'my_value' : 'bar'," + + " 'my_size' : 5" + + " }" + + "}"; + + SearchTemplateRequest request = SearchTemplateRequest.fromXContent(newParser(source)); + assertThat(request.getScript(), equalTo("{\"query\":{\"match\":{\"{{my_field}}\":\"{{my_value}}\"}},\"size\":\"{{my_size}}\"}")); + assertThat(request.getScriptType(), equalTo(ScriptType.INLINE)); + assertThat(request.getScriptParams().size(), equalTo(3)); + assertThat(request.getScriptParams(), hasEntry("my_field", "foo")); + assertThat(request.getScriptParams(), hasEntry("my_value", "bar")); + assertThat(request.getScriptParams(), hasEntry("my_size", 5)); + } + + public void testFromXContentWithMalformedRequest() { + // Unclosed template id + expectThrows(XContentParseException.class, () -> SearchTemplateRequest.fromXContent(newParser("{'id' : 'another_temp }"))); + } + + /** + * Creates a {@link XContentParser} with the given String while replacing single quote to double quotes. + */ + private XContentParser newParser(String s) throws IOException { + assertNotNull(s); + return createParser(JsonXContent.jsonXContent, s.replace("'", "\"")); + } +} diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateResponseTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateResponseTests.java new file mode 100644 index 0000000000000..53f5d1d8f842e --- /dev/null +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateResponseTests.java @@ -0,0 +1,211 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.mustache; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.text.Text; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.internal.InternalSearchResponse; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.io.IOException; +import java.util.Collections; +import java.util.function.Predicate; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; + +public class SearchTemplateResponseTests extends AbstractXContentTestCase { + + @Override + protected SearchTemplateResponse createTestInstance() { + SearchTemplateResponse response = new SearchTemplateResponse(); + if (randomBoolean()) { + response.setResponse(createSearchResponse()); + } else { + response.setSource(createSource()); + } + return response; + } + + @Override + protected SearchTemplateResponse doParseInstance(XContentParser parser) throws IOException { + return SearchTemplateResponse.fromXContent(parser); + } + + /** + * For simplicity we create a minimal response, as there is already a dedicated + * test class for search response parsing and serialization. + */ + private static SearchResponse createSearchResponse() { + long tookInMillis = randomNonNegativeLong(); + int totalShards = randomIntBetween(1, Integer.MAX_VALUE); + int successfulShards = randomIntBetween(0, totalShards); + int skippedShards = randomIntBetween(0, totalShards); + InternalSearchResponse internalSearchResponse = InternalSearchResponse.empty(); + + return new SearchResponse(internalSearchResponse, null, totalShards, successfulShards, + skippedShards, tookInMillis, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY); + } + + private static BytesReference createSource() { + try { + XContentBuilder source = XContentFactory.jsonBuilder() + .startObject() + .startObject("query") + .startObject("match") + .field(randomAlphaOfLength(5), randomAlphaOfLength(10)) + .endObject() + .endObject() + .endObject(); + return BytesReference.bytes(source); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + String templateOutputField = SearchTemplateResponse.TEMPLATE_OUTPUT_FIELD.getPreferredName(); + return field -> field.equals(templateOutputField) || field.startsWith(templateOutputField + "."); + } + + /** + * Note that we can't rely on normal equals and hashCode checks, since {@link SearchResponse} doesn't + * currently implement equals and hashCode. Instead, we compare the template outputs for equality, + * and perform some sanity checks on the search response instances. + */ + @Override + protected void assertEqualInstances(SearchTemplateResponse expectedInstance, SearchTemplateResponse newInstance) { + assertNotSame(newInstance, expectedInstance); + + BytesReference expectedSource = expectedInstance.getSource(); + BytesReference newSource = newInstance.getSource(); + assertEquals(expectedSource == null, newSource == null); + if (expectedSource != null) { + try { + assertToXContentEquivalent(expectedSource, newSource, XContentType.JSON); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + assertEquals(expectedInstance.hasResponse(), newInstance.hasResponse()); + if (expectedInstance.hasResponse()) { + SearchResponse expectedResponse = expectedInstance.getResponse(); + SearchResponse newResponse = newInstance.getResponse(); + + assertEquals(expectedResponse.getHits().totalHits, newResponse.getHits().totalHits); + assertEquals(expectedResponse.getHits().getMaxScore(), newResponse.getHits().getMaxScore(), 0.0001); + } + } + + @Override + protected boolean supportsUnknownFields() { + return true; + } + + public void testSourceToXContent() throws IOException { + SearchTemplateResponse response = new SearchTemplateResponse(); + + XContentBuilder source = XContentFactory.jsonBuilder() + .startObject() + .startObject("query") + .startObject("terms") + .field("status", new String[]{"pending", "published"}) + .endObject() + .endObject() + .endObject(); + response.setSource(BytesReference.bytes(source)); + + XContentType contentType = randomFrom(XContentType.values()); + XContentBuilder expectedResponse = XContentFactory.contentBuilder(contentType) + .startObject() + .startObject("template_output") + .startObject("query") + .startObject("terms") + .field("status", new String[]{"pending", "published"}) + .endObject() + .endObject() + .endObject() + .endObject(); + + XContentBuilder actualResponse = XContentFactory.contentBuilder(contentType); + response.toXContent(actualResponse, ToXContent.EMPTY_PARAMS); + + assertToXContentEquivalent( + BytesReference.bytes(expectedResponse), + BytesReference.bytes(actualResponse), + contentType); + } + + public void testSearchResponseToXContent() throws IOException { + SearchHit hit = new SearchHit(1, "id", new Text("type"), Collections.emptyMap()); + hit.score(2.0f); + SearchHit[] hits = new SearchHit[] { hit }; + + InternalSearchResponse internalSearchResponse = new InternalSearchResponse( + new SearchHits(hits, 100, 1.5f), null, null, null, false, null, 1); + SearchResponse searchResponse = new SearchResponse(internalSearchResponse, null, + 0, 0, 0, 0, ShardSearchFailure.EMPTY_ARRAY, SearchResponse.Clusters.EMPTY); + + SearchTemplateResponse response = new SearchTemplateResponse(); + response.setResponse(searchResponse); + + XContentType contentType = randomFrom(XContentType.values()); + XContentBuilder expectedResponse = XContentFactory.contentBuilder(contentType) + .startObject() + .field("took", 0) + .field("timed_out", false) + .startObject("_shards") + .field("total", 0) + .field("successful", 0) + .field("skipped", 0) + .field("failed", 0) + .endObject() + .startObject("hits") + .field("total", 100) + .field("max_score", 1.5F) + .startArray("hits") + .startObject() + .field("_type", "type") + .field("_id", "id") + .field("_score", 2.0F) + .endObject() + .endArray() + .endObject() + .endObject(); + + XContentBuilder actualResponse = XContentFactory.contentBuilder(contentType); + response.toXContent(actualResponse, ToXContent.EMPTY_PARAMS); + + assertToXContentEquivalent( + BytesReference.bytes(expectedResponse), + BytesReference.bytes(actualResponse), + contentType); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java index fa851e9c6d802..d534af5789448 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java @@ -82,7 +82,7 @@ private RandomSearchRequestGenerator() {} * @param randomSearchSourceBuilder builds a random {@link SearchSourceBuilder}. You can use * {@link #randomSearchSourceBuilder(Supplier, Supplier, Supplier, Supplier, Supplier)}. */ - public static SearchRequest randomSearchRequest(Supplier randomSearchSourceBuilder) throws IOException { + public static SearchRequest randomSearchRequest(Supplier randomSearchSourceBuilder) { SearchRequest searchRequest = new SearchRequest(); searchRequest.allowPartialSearchResults(true); if (randomBoolean()) { From 2cb71d0947947c94ccd78741283df6c966aad1db Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Tue, 15 May 2018 15:03:08 -0600 Subject: [PATCH 05/34] Refactor IndicesOptions to not be byte-based (#30586) * Refactor IndicesOptions to not be byte-based This refactors IndicesOptions to be enum/enummap based rather than using a byte as a bitmap for each of the options. This is necessary because we'd like to add additional options, but we ran out of bits. Backwards compatibility is kept for earlier versions so the option serialization does not change the options. Relates sort of to #30188 --- .../action/support/IndicesOptions.java | 307 ++++++++++++------ .../common/io/stream/StreamInput.java | 18 + .../common/io/stream/StreamOutput.java | 11 + .../ClusterSearchShardsRequestTests.java | 2 +- .../action/support/IndicesOptionsTests.java | 80 +++++ 5 files changed, 323 insertions(+), 95 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java index 64c26d6b94aa5..afcb73e9b1cbd 100644 --- a/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java +++ b/server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java @@ -19,12 +19,17 @@ package org.elasticsearch.action.support; +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.rest.RestRequest; import java.io.IOException; +import java.util.Collection; +import java.util.EnumSet; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeStringArrayValue; @@ -35,41 +40,155 @@ */ public class IndicesOptions { - private static final IndicesOptions[] VALUES; + public enum WildcardStates { + OPEN, + CLOSED; - private static final byte IGNORE_UNAVAILABLE = 1; - private static final byte ALLOW_NO_INDICES = 2; - private static final byte EXPAND_WILDCARDS_OPEN = 4; - private static final byte EXPAND_WILDCARDS_CLOSED = 8; - private static final byte FORBID_ALIASES_TO_MULTIPLE_INDICES = 16; - private static final byte FORBID_CLOSED_INDICES = 32; - private static final byte IGNORE_ALIASES = 64; + public static final EnumSet NONE = EnumSet.noneOf(WildcardStates.class); - private static final byte STRICT_EXPAND_OPEN = 6; - private static final byte LENIENT_EXPAND_OPEN = 7; - private static final byte STRICT_EXPAND_OPEN_CLOSED = 14; - private static final byte STRICT_EXPAND_OPEN_FORBID_CLOSED = 38; - private static final byte STRICT_SINGLE_INDEX_NO_EXPAND_FORBID_CLOSED = 48; + public static EnumSet parseParameter(Object value, EnumSet defaultStates) { + if (value == null) { + return defaultStates; + } - static { - short max = 1 << 7; - VALUES = new IndicesOptions[max]; - for (short id = 0; id < max; id++) { - VALUES[id] = new IndicesOptions((byte)id); + Set states = new HashSet<>(); + String[] wildcards = nodeStringArrayValue(value); + for (String wildcard : wildcards) { + if ("open".equals(wildcard)) { + states.add(OPEN); + } else if ("closed".equals(wildcard)) { + states.add(CLOSED); + } else if ("none".equals(wildcard)) { + states.clear(); + } else if ("all".equals(wildcard)) { + states.add(OPEN); + states.add(CLOSED); + } else { + throw new IllegalArgumentException("No valid expand wildcard value [" + wildcard + "]"); + } + } + + return states.isEmpty() ? NONE : EnumSet.copyOf(states); } } - private final byte id; + public enum Option { + IGNORE_UNAVAILABLE, + IGNORE_ALIASES, + ALLOW_NO_INDICES, + FORBID_ALIASES_TO_MULTIPLE_INDICES, + FORBID_CLOSED_INDICES; + + public static final EnumSet