[Endpoint] Fix endpoint tests with data streams

src/es_archiver/actions/load.ts

@@ -49,13 +49,15 @@ const pipeline = (...streams: Readable[]) =>
 export async function loadAction({
   name,
   skipExisting,
+  useCreate,
   client,
   dataDir,
   log,
   kbnClient,
 }: {
   name: string;
   skipExisting: boolean;
+  useCreate: boolean;
   client: Client;
   dataDir: string;
   log: ToolingLog;
@@ -87,7 +89,7 @@ export async function loadAction({
   await createPromiseFromStreams([
     recordStream,
     createCreateIndexStream({ client, stats, skipExisting, log }),
-    createIndexDocRecordsStream(client, stats, progress),
+    createIndexDocRecordsStream(client, stats, progress, useCreate),
   ]);
 
   progress.deactivate();

src/es_archiver/es_archiver.ts

@@ -80,12 +80,20 @@ export class EsArchiver {
    *  @param {Object} options
    *  @property {Boolean} options.skipExisting - should existing indices
    *                                           be ignored or overwritten
+   *  @property {Boolean} options.useCreate - use a create operation instead of index for documents
    *  @return Promise<Stats>
    */
-  async load(name: string, { skipExisting = false }: { skipExisting?: boolean } = {}) {
+  async load(
+    name: string,
+    {
+      skipExisting = false,
+      useCreate = false,
+    }: { skipExisting?: boolean; useCreate?: boolean } = {}
+  ) {
     return await loadAction({
       name,
       skipExisting: !!skipExisting,
+      useCreate: !!useCreate,
       client: this.client,
       dataDir: this.dataDir,
       log: this.log,

src/es_archiver/lib/docs/index_doc_records_stream.ts

@@ -22,15 +22,20 @@ import { Writable } from 'stream';
 import { Stats } from '../stats';
 import { Progress } from '../progress';
 
-export function createIndexDocRecordsStream(client: Client, stats: Stats, progress: Progress) {
+export function createIndexDocRecordsStream(
+  client: Client,
+  stats: Stats,
+  progress: Progress,
+  useCreate: boolean = false
+) {
   async function indexDocs(docs: any[]) {
     const body: any[] = [];
-
+    const operation = useCreate === true ? 'create' : 'index';
     docs.forEach((doc) => {
       stats.indexedDoc(doc.index);
       body.push(
         {
-          index: {
+          [operation]: {
             _index: doc.index,
             _id: doc.id,
           },

x-pack/test/api_integration/apis/endpoint/alerts/index.ts

@@ -6,7 +6,7 @@
 import expect from '@kbn/expect/expect.js';
 import { FtrProviderContext } from '../../../ftr_provider_context';
 import { AlertData } from '../../../../../plugins/security_solution/common/endpoint_alerts/types';
-import { AlertId } from '../../../../../plugins/security_solution/server/endpoint/alerts/handlers/lib/index';
+import { deleteEventsStream, deleteMetadataStream } from '../data_stream_helper';
 
 /**
  * The number of alert documents in the es archive.
@@ -66,26 +66,25 @@ export default function ({ getService }: FtrProviderContext) {
   const nextPrevPrefixOrder = 'order=desc';
   const nextPrevPrefixPageSize = 'page_size=10';
   const nextPrevPrefix = `${nextPrevPrefixQuery}&${nextPrevPrefixDateRange}&${nextPrevPrefixSort}&${nextPrevPrefixOrder}&${nextPrevPrefixPageSize}`;
-  const alertIndex = 'events-endpoint-1';
 
   let nullableEventId = '';
 
-  // SKIPPED as it is failing ES PROMOTION: https://github.com/elastic/kibana/issues/68613
-  describe.skip('Endpoint alert API', () => {
+  describe('Endpoint alert API', () => {
     describe('when data is in elasticsearch', () => {
       before(async () => {
-        await esArchiver.load('endpoint/alerts/api_feature');
-        await esArchiver.load('endpoint/alerts/host_api_feature');
+        await esArchiver.load('endpoint/alerts/api_feature', { useCreate: true });
+        await esArchiver.load('endpoint/alerts/host_api_feature', { useCreate: true });
         const res = await es.search({
-          index: alertIndex,
+          index: 'events-endpoint-*',
           body: ES_QUERY_MISSING,
         });
         nullableEventId = res.hits.hits[0]._source.event.id;
       });
 
       after(async () => {
-        await esArchiver.unload('endpoint/alerts/api_feature');
-        await esArchiver.unload('endpoint/alerts/host_api_feature');
+        // the endpoint uses data streams and es archiver does not support deleting them at the moment so we need
+        // to do it manually
+        await Promise.all([deleteEventsStream(getService), deleteMetadataStream(getService)]);
       });
 
       it('should not support POST requests', async () => {
@@ -375,40 +374,6 @@ export default function ({ getService }: FtrProviderContext) {
         expect(body.result_from_index).to.eql(0);
       });
 
-      it('should return alert details by id, getting last alert', async () => {
-        const documentID = new AlertId(alertIndex, 'zbNm0HABdD75WLjLYgcB');
-        const prevDocumentID = new AlertId(alertIndex, '2rNm0HABdD75WLjLYgcU');
-        const { body } = await supertest
-          .get(`/api/endpoint/alerts/${documentID.toString()}`)
-          .set('kbn-xsrf', 'xxx')
-          .expect(200);
-        expect(body.id).to.eql(documentID.toString());
-        expect(body.prev).to.eql(`/api/endpoint/alerts/${prevDocumentID.toString()}`);
-        expect(body.next).to.eql(null); // last alert, no more beyond this
-        expect(body.state.host_metadata.host.id).to.eql(body.host.id);
-      });
-
-      it('should return alert details by id, getting first alert', async () => {
-        const documentID = new AlertId(alertIndex, 'p7Nm0HABdD75WLjLYghv');
-        const nextDocumentID = new AlertId(alertIndex, 'mbNm0HABdD75WLjLYgho');
-        const { body } = await supertest
-          .get(`/api/endpoint/alerts/${documentID.toString()}`)
-          .set('kbn-xsrf', 'xxx')
-          .expect(200);
-        expect(body.id).to.eql(documentID.toString());
-        expect(body.next).to.eql(`/api/endpoint/alerts/${nextDocumentID.toString()}`);
-        expect(body.prev).to.eql(null); // first alert, no more before this
-      });
-
-      it('should return 404 when alert is not found', async () => {
-        const documentID = new AlertId(alertIndex, 'does-not-exit');
-
-        await supertest
-          .get(`/api/endpoint/alerts/${documentID.toString()}`)
-          .set('kbn-xsrf', 'xxx')
-          .expect(404);
-      });
-
       it('should return 400 when alert id is not valid', async () => {
         await supertest
           .get('/api/endpoint/alerts/does-not-exist')

x-pack/test/api_integration/apis/endpoint/data_stream_helper.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { Client } from '@elastic/elasticsearch';
+
+export async function deleteDataStream(getService: (serviceName: 'es') => Client, index: string) {
+  const client = getService('es');
+  await client.transport.request(
+    {
+      method: 'DELETE',
+      path: `_data_stream/${index}`,
+    },
+    {
+      ignore: [404],
+    }
+  );
+}
+
+export async function deleteMetadataStream(getService: (serviceName: 'es') => Client) {
+  await deleteDataStream(getService, 'metrics-endpoint.metadata-*');
+}
+
+export async function deleteEventsStream(getService: (serviceName: 'es') => Client) {
+  await deleteDataStream(getService, 'events-endpoint-*');
+}
+
+export async function deletePolicyStream(getService: (serviceName: 'es') => Client) {
+  await deleteDataStream(getService, 'metrics-endpoint.policy-*');
+}

x-pack/test/api_integration/apis/endpoint/metadata.ts

@@ -5,6 +5,7 @@
  */
 import expect from '@kbn/expect/expect.js';
 import { FtrProviderContext } from '../../ftr_provider_context';
+import { deleteMetadataStream } from './data_stream_helper';
 
 /**
  * The number of host documents in the es archive.
@@ -14,11 +15,12 @@ const numberOfHostsInFixture = 3;
 export default function ({ getService }: FtrProviderContext) {
   const esArchiver = getService('esArchiver');
   const supertest = getService('supertest');
-  // SKIPPED as it is failing on ES PROMOTION: https://github.com/elastic/kibana/issues/68584
-  describe.skip('test metadata api', () => {
+  describe('test metadata api', () => {
     describe('POST /api/endpoint/metadata when index is empty', () => {
       it('metadata api should return empty result when index is empty', async () => {
-        await esArchiver.unload('endpoint/metadata/api_feature');
+        // the endpoint uses data streams and es archiver does not support deleting them at the moment so we need
+        // to do it manually
+        await deleteMetadataStream(getService);
         const { body } = await supertest
           .post('/api/endpoint/metadata')
           .set('kbn-xsrf', 'xxx')
@@ -32,8 +34,12 @@ export default function ({ getService }: FtrProviderContext) {
     });
 
     describe('POST /api/endpoint/metadata when index is not empty', () => {
-      before(() => esArchiver.load('endpoint/metadata/api_feature'));
-      after(() => esArchiver.unload('endpoint/metadata/api_feature'));
+      before(
+        async () => await esArchiver.load('endpoint/metadata/api_feature', { useCreate: true })
+      );
+      // the endpoint uses data streams and es archiver does not support deleting them at the moment so we need
+      // to do it manually
+      after(async () => await deleteMetadataStream(getService));
       it('metadata api should return one entry for each host with default paging', async () => {
         const { body } = await supertest
           .post('/api/endpoint/metadata')

x-pack/test/api_integration/apis/endpoint/policy.ts

@@ -6,16 +6,18 @@
 
 import expect from '@kbn/expect/expect.js';
 import { FtrProviderContext } from '../../ftr_provider_context';
+import { deletePolicyStream } from './data_stream_helper';
 
 export default function ({ getService }: FtrProviderContext) {
   const esArchiver = getService('esArchiver');
   const supertest = getService('supertest');
-  // SKIPPED as it is failing ES PROMOTION: https://github.com/elastic/kibana/issues/68638
-  describe.skip('Endpoint policy api', () => {
+  describe('Endpoint policy api', () => {
     describe('GET /api/endpoint/policy_response', () => {
-      before(async () => await esArchiver.load('endpoint/policy'));
+      before(async () => await esArchiver.load('endpoint/policy', { useCreate: true }));
 
-      after(async () => await esArchiver.unload('endpoint/policy'));
+      // the endpoint uses data streams and es archiver does not support deleting them at the moment so we need
+      // to do it manually
+      after(async () => await deletePolicyStream(getService));
 
       it('should return one policy response for host', async () => {
         const expectedHostId = '4f3b9858-a96d-49d8-a326-230d7763d767';

x-pack/test/api_integration/apis/endpoint/resolver.ts

@@ -216,6 +216,7 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC
     });
     after(async () => {
       await resolver.deleteTrees(resolverTrees);
+      // this unload is for an endgame-* index so it does not use data streams
       await esArchiver.unload('endpoint/resolver/api_feature');
     });