Revert "Fix Active Directory tests (elastic#47266)"

This reverts commit b917b57.

x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ADLdapUserSearchSessionFactoryTests.java

@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.security.authc.ldap;
 
 import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -14,8 +15,10 @@
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
+import org.elasticsearch.xpack.core.security.authc.ldap.support.LdapSearchScope;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.security.authc.ldap.support.LdapSession;
+import org.elasticsearch.xpack.security.authc.ldap.support.LdapTestCase;
 import org.elasticsearch.xpack.security.authc.ldap.support.SessionFactory;
 import org.junit.After;
 import org.junit.Before;
@@ -46,7 +49,7 @@ public void init() throws Exception {
 
         globalSettings = Settings.builder()
             .put("path.home", createTempDir())
-            .put("xpack.security.authc.realms.ldap.ad-as-ldap-test.ssl.certificate_authorities", certPath)
+            .put("xpack.security.authc.realms.active_directory.ad.ssl.certificate_authorities", certPath)
             .build();
         sslService = new SSLService(globalSettings, env);
         threadPool = new TestThreadPool("ADLdapUserSearchSessionFactoryTests");
@@ -57,12 +60,15 @@ public void shutdown() {
         terminate(threadPool);
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/35738")
     public void testUserSearchWithActiveDirectory() throws Exception {
         String groupSearchBase = "DC=ad,DC=test,DC=elasticsearch,DC=com";
         String userSearchBase = "CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com";
         Settings settings = Settings.builder()
-                .put("url", ActiveDirectorySessionFactoryTests.AD_LDAP_URL)
-                .put("group_search.base_dn", groupSearchBase)
+                .put(LdapTestCase.buildLdapSettings(
+                        new String[] { ActiveDirectorySessionFactoryTests.AD_LDAP_URL },
+                        Strings.EMPTY_ARRAY, groupSearchBase, LdapSearchScope.SUB_TREE, null,
+                        true))
                 .put("user_search.base_dn", userSearchBase)
                 .put("bind_dn", "[email protected]")
                 .put("bind_password", ActiveDirectorySessionFactoryTests.PASSWORD)
@@ -73,13 +79,13 @@ public void testUserSearchWithActiveDirectory() throws Exception {
         Settings.Builder builder = Settings.builder()
                 .put(globalSettings);
         settings.keySet().forEach(k -> {
-            builder.copy("xpack.security.authc.realms.ldap.ad-as-ldap-test." + k, k, settings);
+            builder.copy("xpack.security.authc.realms.ad-as-ldap-test." + k, k, settings);
 
         });
         Settings fullSettings = builder.build();
         sslService = new SSLService(fullSettings, TestEnvironment.newEnvironment(fullSettings));
-        RealmConfig config = new RealmConfig(new RealmConfig.RealmIdentifier("ldap", "ad-as-ldap-test"), fullSettings,
-                TestEnvironment.newEnvironment(fullSettings), new ThreadContext(fullSettings));
+        RealmConfig config = new RealmConfig(new RealmConfig.RealmIdentifier("ad", "ad-as-ldap-test"), globalSettings,
+                TestEnvironment.newEnvironment(globalSettings), new ThreadContext(globalSettings));
         LdapUserSearchSessionFactory sessionFactory = getLdapUserSearchSessionFactory(config, sslService, threadPool);
 
         String user = "Bruce Banner";

x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractActiveDirectoryTestCase.java

@@ -84,10 +84,10 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO
         Settings.Builder builder = Settings.builder().put("path.home", createTempDir());
 
         // fake realms so ssl will get loaded
-        builder.putList("xpack.security.authc.realms.active_directory.foo.ssl.certificate_authorities", certificatePaths);
-        builder.put("xpack.security.authc.realms.active_directory.foo.ssl.verification_mode", VerificationMode.FULL);
-        builder.putList("xpack.security.authc.realms.active_directory.bar.ssl.certificate_authorities", certificatePaths);
-        builder.put("xpack.security.authc.realms.active_directory.bar.ssl.verification_mode", VerificationMode.CERTIFICATE);
+        builder.putList("xpack.security.authc.realms.foo.ssl.certificate_authorities", certificatePaths);
+        builder.put("xpack.security.authc.realms.foo.ssl.verification_mode", VerificationMode.FULL);
+        builder.putList("xpack.security.authc.realms.bar.ssl.certificate_authorities", certificatePaths);
+        builder.put("xpack.security.authc.realms.bar.ssl.verification_mode", VerificationMode.CERTIFICATE);
         globalSettings = builder.build();
         Environment environment = TestEnvironment.newEnvironment(globalSettings);
         sslService = new SSLService(globalSettings, environment);

x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java

@@ -24,6 +24,7 @@
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingResponse;
 import org.elasticsearch.xpack.core.security.authc.ldap.ActiveDirectorySessionFactorySettings;
+import org.elasticsearch.xpack.core.security.authc.ldap.LdapRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 import org.junit.After;
@@ -48,7 +49,6 @@
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.xpack.core.security.authc.RealmSettings.getFullSettingKey;
 import static org.elasticsearch.xpack.core.security.authc.ldap.support.LdapSearchScope.ONE_LEVEL;
 import static org.elasticsearch.xpack.core.security.authc.ldap.support.LdapSearchScope.SUB_TREE;
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.BASIC_AUTH_HEADER;
@@ -65,8 +65,7 @@
  */
 public abstract class AbstractAdLdapRealmTestCase extends SecurityIntegTestCase {
 
-    public static final String XPACK_SECURITY_AUTHC_REALMS_AD_EXTERNAL = "xpack.security.authc.realms.active_directory.external";
-    public static final String XPACK_SECURITY_AUTHC_REALMS_LDAP_EXTERNAL = "xpack.security.authc.realms.ldap.external";
+    public static final String XPACK_SECURITY_AUTHC_REALMS_EXTERNAL = "xpack.security.authc.realms.external";
     public static final String PASSWORD = AbstractActiveDirectoryTestCase.PASSWORD;
     public static final String ASGARDIAN_INDEX = "gods";
     public static final String PHILANTHROPISTS_INDEX = "philanthropists";
@@ -126,13 +125,13 @@ protected Settings nodeSettings(int nodeOrdinal) {
         Settings.Builder builder = Settings.builder();
         // don't use filter since it returns a prefixed secure setting instead of mock!
         Settings settingsToAdd = super.nodeSettings(nodeOrdinal);
-        builder.put(settingsToAdd.filter(k -> k.startsWith("xpack.security.transport.ssl.") == false), false);
+        builder.put(settingsToAdd.filter(k -> k.startsWith("xpack.transport.security.ssl.") == false), false);
         MockSecureSettings mockSecureSettings = (MockSecureSettings) Settings.builder().put(settingsToAdd).getSecureSettings();
         if (mockSecureSettings != null) {
             MockSecureSettings filteredSecureSettings = new MockSecureSettings();
             builder.setSecureSettings(filteredSecureSettings);
             for (String secureSetting : mockSecureSettings.getSettingNames()) {
-                if (secureSetting.startsWith("xpack.security.transport.ssl.") == false) {
+                if (secureSetting.startsWith("xpack.transport.security.ssl.") == false) {
                     SecureString secureString = mockSecureSettings.getString(secureSetting);
                     if (secureString == null) {
                         final byte[] fileBytes;
@@ -165,7 +164,7 @@ protected Settings buildRealmSettings(RealmConfig realm, List<RoleMappingEntry>
         certificateAuthorities) {
         Settings.Builder builder = Settings.builder();
         builder.put(realm.buildSettings(certificateAuthorities));
-        configureFileRoleMappings(builder, realm.type, roleMappingEntries);
+        configureFileRoleMappings(builder, roleMappingEntries);
         return builder.build();
     }
 
@@ -213,11 +212,11 @@ private List<String> getRoleMappingContent(Function<RoleMappingEntry, String> co
                 .collect(Collectors.toList());
     }
 
-    protected final void configureFileRoleMappings(Settings.Builder builder, String realmType, List<RoleMappingEntry> mappings) {
+    protected final void configureFileRoleMappings(Settings.Builder builder, List<RoleMappingEntry> mappings) {
         String content = getRoleMappingContent(RoleMappingEntry::getFileContent, mappings).stream().collect(Collectors.joining("\n"));
         Path nodeFiles = createTempDir();
         String file = writeFile(nodeFiles, "role_mapping.yml", content);
-        builder.put("xpack.security.authc.realms." + realmType + ".external.files.role_mapping", file);
+        builder.put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".files.role_mapping", file);
     }
 
     @Override
@@ -288,11 +287,11 @@ protected static String userHeader(String username, String password) {
 
     private void addSslSettingsForKeyPair(Settings.Builder builder, Path key, String keyPassphrase, Path cert,
                                           List<String> certificateAuthorities) {
-        builder.put("xpack.security.transport.ssl.key", key)
-            .put("xpack.security.transport.ssl.key_passphrase", keyPassphrase)
-            .put("xpack.security.transport.ssl.verification_mode", "certificate")
-            .put("xpack.security.transport.ssl.certificate", cert)
-            .putList("xpack.security.transport.ssl.certificate_authorities", certificateAuthorities);
+        builder.put("xpack.transport.security.ssl.key", key)
+            .put("xpack.transport.security.ssl.key_passphrase", keyPassphrase)
+            .put("xpack.transport.security.ssl.verification_mode", "certificate")
+            .put("xpack.transport.security.ssl.certificate", cert)
+            .putList("xpack.transport.security.ssl.certificate_authorities", certificateAuthorities);
     }
 
     /**
@@ -377,55 +376,57 @@ enum RealmConfig {
 
         AD(false, AD_ROLE_MAPPING,
                 Settings.builder()
-                    .put(XPACK_SECURITY_AUTHC_REALMS_AD_EXTERNAL + ".domain_name", ActiveDirectorySessionFactoryTests.AD_DOMAIN)
-                    .put(XPACK_SECURITY_AUTHC_REALMS_AD_EXTERNAL
+                        .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".type", LdapRealmSettings.AD_TYPE)
+                        .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".domain_name", ActiveDirectorySessionFactoryTests.AD_DOMAIN)
+                        .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL
                                 + ".group_search.base_dn", "CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com")
-                    .put(XPACK_SECURITY_AUTHC_REALMS_AD_EXTERNAL + ".group_search.scope", randomBoolean() ? SUB_TREE : ONE_LEVEL)
-                    .put(XPACK_SECURITY_AUTHC_REALMS_AD_EXTERNAL + ".url", ActiveDirectorySessionFactoryTests.AD_LDAP_URL)
-                    .put(XPACK_SECURITY_AUTHC_REALMS_AD_EXTERNAL + ".follow_referrals",
+                        .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".group_search.scope", randomBoolean() ? SUB_TREE : ONE_LEVEL)
+                        .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".url", ActiveDirectorySessionFactoryTests.AD_LDAP_URL)
+                        .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".follow_referrals",
                                 ActiveDirectorySessionFactoryTests.FOLLOW_REFERRALS)
-                    .put(getFullSettingKey("external",ActiveDirectorySessionFactorySettings.AD_LDAP_PORT_SETTING), AD_LDAP_PORT)
-                    .put(getFullSettingKey("external",ActiveDirectorySessionFactorySettings.AD_LDAPS_PORT_SETTING), AD_LDAPS_PORT)
-                    .put(getFullSettingKey("external",ActiveDirectorySessionFactorySettings.AD_GC_LDAP_PORT_SETTING), AD_GC_LDAP_PORT)
-                    .put(getFullSettingKey("external",ActiveDirectorySessionFactorySettings.AD_GC_LDAPS_PORT_SETTING), AD_GC_LDAPS_PORT)
-                    .build(),
-            "active_directory"),
+                        .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + "." +
+                                ActiveDirectorySessionFactorySettings.AD_LDAP_PORT_SETTING.getKey(), AD_LDAP_PORT)
+                        .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + "." +
+                                ActiveDirectorySessionFactorySettings.AD_LDAPS_PORT_SETTING.getKey(), AD_LDAPS_PORT)
+                        .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + "." +
+                                ActiveDirectorySessionFactorySettings.AD_GC_LDAP_PORT_SETTING.getKey(), AD_GC_LDAP_PORT)
+                        .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + "." +
+                                ActiveDirectorySessionFactorySettings.AD_GC_LDAPS_PORT_SETTING.getKey(), AD_GC_LDAPS_PORT)
+                        .build()),
 
         AD_LDAP_GROUPS_FROM_SEARCH(true, AD_ROLE_MAPPING,
                 Settings.builder()
-                    .put(XPACK_SECURITY_AUTHC_REALMS_LDAP_EXTERNAL + ".url", ActiveDirectorySessionFactoryTests.AD_LDAP_URL)
-                    .put(XPACK_SECURITY_AUTHC_REALMS_LDAP_EXTERNAL
+                        .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".type", LdapRealmSettings.LDAP_TYPE)
+                        .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".url", ActiveDirectorySessionFactoryTests.AD_LDAP_URL)
+                        .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL
                                 + ".group_search.base_dn", "CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com")
-                    .put(XPACK_SECURITY_AUTHC_REALMS_LDAP_EXTERNAL + ".group_search.scope", randomBoolean() ? SUB_TREE : ONE_LEVEL)
-                    .putList(XPACK_SECURITY_AUTHC_REALMS_LDAP_EXTERNAL + ".user_dn_templates",
+                        .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".group_search.scope", randomBoolean() ? SUB_TREE : ONE_LEVEL)
+                        .putList(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".user_dn_templates",
                                 "cn={0},CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com")
-                    .put(XPACK_SECURITY_AUTHC_REALMS_LDAP_EXTERNAL + ".follow_referrals",
+                        .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".follow_referrals",
                                 ActiveDirectorySessionFactoryTests.FOLLOW_REFERRALS)
-                    .build(),
-            "ldap"),
+                        .build()),
 
         AD_LDAP_GROUPS_FROM_ATTRIBUTE(true, AD_ROLE_MAPPING,
                 Settings.builder()
-                    .put(XPACK_SECURITY_AUTHC_REALMS_LDAP_EXTERNAL + ".url", ActiveDirectorySessionFactoryTests.AD_LDAP_URL)
-                    .putList(XPACK_SECURITY_AUTHC_REALMS_LDAP_EXTERNAL + ".user_dn_templates",
+                        .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".type", LdapRealmSettings.LDAP_TYPE)
+                        .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".url", ActiveDirectorySessionFactoryTests.AD_LDAP_URL)
+                        .putList(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".user_dn_templates",
                                 "cn={0},CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com")
-                    .put(XPACK_SECURITY_AUTHC_REALMS_LDAP_EXTERNAL + ".follow_referrals",
+                        .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".follow_referrals",
                                 ActiveDirectorySessionFactoryTests.FOLLOW_REFERRALS)
-                    .build(),
-            "ldap");
+                        .build());
 
-        final String type;
         final boolean mapGroupsAsRoles;
         final boolean loginWithCommonName;
         private final RoleMappingEntry[] roleMappings;
         final Settings settings;
 
-        RealmConfig(boolean loginWithCommonName, RoleMappingEntry[] roleMappings, Settings settings, String type) {
+        RealmConfig(boolean loginWithCommonName, RoleMappingEntry[] roleMappings, Settings settings) {
             this.settings = settings;
             this.loginWithCommonName = loginWithCommonName;
             this.roleMappings = roleMappings;
             this.mapGroupsAsRoles = randomBoolean();
-            this.type = type;
         }
 
         public Settings buildSettings(List<String> certificateAuthorities) {
@@ -435,11 +436,11 @@ public Settings buildSettings(List<String> certificateAuthorities) {
 
         protected Settings buildSettings(List<String> certificateAuthorities, int order) {
             Settings.Builder builder = Settings.builder()
-                .put("xpack.security.authc.realms." + type + ".external.order", order)
-                .put("xpack.security.authc.realms." + type + ".external.hostname_verification", false)
-                .put("xpack.security.authc.realms." + type + ".external.unmapped_groups_as_roles", mapGroupsAsRoles)
+                .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".order", order)
+                .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".hostname_verification", false)
+                .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".unmapped_groups_as_roles", mapGroupsAsRoles)
                 .put(this.settings)
-                .putList("xpack.security.authc.realms." + type + ".external.ssl.certificate_authorities", certificateAuthorities);
+                .putList(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".ssl.certificate_authorities", certificateAuthorities);
             return builder.build();
         }
 

x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryGroupsResolverTests.java

@@ -34,6 +34,7 @@ public void setReferralFollowing() {
         ldapConnection.getConnectionOptions().setFollowReferrals(AbstractActiveDirectoryTestCase.FOLLOW_REFERRALS);
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/35738")
     public void testResolveSubTree() throws Exception {
         Settings settings = Settings.builder()
                 .put("xpack.security.authc.realms.active_directory.ad.group_search.scope", LdapSearchScope.SUB_TREE)