Enable logs for intermittent test failure

I have not been able to reproduce the failing
test scenario locally for elastic#38408 and there are other similar
tests which are running fine in the same test class.
I am re-enabling the test with additional logs so
that we can debug further on what's happening.
I will keep the issue open for now and look out for the builds
to see if there are any related failures.

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java

@@ -6,6 +6,7 @@
 
 package org.elasticsearch.xpack.security.authc;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.search.SearchResponse;
@@ -21,6 +22,7 @@
 import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSource;
 import org.elasticsearch.test.SecuritySettingsSourceField;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.security.action.CreateApiKeyResponse;
 import org.elasticsearch.xpack.core.security.action.GetApiKeyRequest;
@@ -54,8 +56,11 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.isIn;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
 
+@TestLogging("org.elasticsearch.xpack.security.authc.ApiKeyService:TRACE")
 public class ApiKeyIntegTests extends SecurityIntegTestCase {
 
     @Override
@@ -232,7 +237,6 @@ public void testInvalidateApiKeysForApiKeyName() throws InterruptedException, Ex
         verifyInvalidateResponse(1, responses, invalidateResponse);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/38408")
     public void testGetAndInvalidateApiKeysWithExpiredAndInvalidatedApiKey() throws Exception {
         List<CreateApiKeyResponse> responses = createApiKeys(1, null);
         Instant created = Instant.now();
@@ -249,6 +253,9 @@ public void testGetAndInvalidateApiKeysWithExpiredAndInvalidatedApiKey() throws
             assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L));
             docId.set(searchResponse.getHits().getAt(0).getId());
         });
+        logger.info("searched and found API key with doc id = " + docId.get());
+        assertThat(docId.get(), is(notNullValue()));
+        assertThat(docId.get(), is(responses.get(0).getId()));
 
         // hack doc to modify the expiration time to the week before
         Instant weekBefore = created.minus(8L, ChronoUnit.DAYS);
@@ -259,6 +266,11 @@ public void testGetAndInvalidateApiKeysWithExpiredAndInvalidatedApiKey() throws
         PlainActionFuture<InvalidateApiKeyResponse> listener = new PlainActionFuture<>();
         securityClient.invalidateApiKey(InvalidateApiKeyRequest.usingApiKeyId(responses.get(0).getId()), listener);
         InvalidateApiKeyResponse invalidateResponse = listener.get();
+        if (invalidateResponse.getErrors().isEmpty() == false) {
+            logger.error("error occurred while invalidating API key by id : " + invalidateResponse.getErrors().stream()
+                    .map(ElasticsearchException::getMessage)
+                    .collect(Collectors.joining(", ")));
+        }
         verifyInvalidateResponse(1, responses, invalidateResponse);
 
         // try again
@@ -303,6 +315,9 @@ public void testInvalidatedApiKeysDeletedByRemover() throws Exception {
             assertThat(searchResponse.getHits().getTotalHits().value, equalTo(2L));
             docId.set(searchResponse.getHits().getAt(0).getId());
         });
+        logger.info("searched and found API key with doc id = " + docId.get());
+        assertThat(docId.get(), is(notNullValue()));
+        assertThat(docId.get(), isIn(responses.stream().map(CreateApiKeyResponse::getId).collect(Collectors.toList())));
 
         AtomicBoolean deleteTriggered = new AtomicBoolean(false);
         assertBusy(() -> {
@@ -333,6 +348,9 @@ public void testExpiredApiKeysDeletedAfter1Week() throws Exception {
             assertThat(searchResponse.getHits().getTotalHits().value, equalTo(2L));
             docId.set(searchResponse.getHits().getAt(0).getId());
         });
+        logger.info("searched and found API key with doc id = " + docId.get());
+        assertThat(docId.get(), is(notNullValue()));
+        assertThat(docId.get(), isIn(responses.stream().map(CreateApiKeyResponse::getId).collect(Collectors.toList())));
 
         // hack doc to modify the expiration time to the week before
         Instant weekBefore = created.minus(8L, ChronoUnit.DAYS);
@@ -490,6 +508,7 @@ private List<CreateApiKeyResponse> createApiKeys(int noOfApiKeys, TimeValue expi
             assertNotNull(response.getKey());
             responses.add(response);
         }
+        assertThat(responses.size(), is(noOfApiKeys));
         return responses;
     }
 }