[Automatic Import] Add more FTR tests (elastic#203772)

(cherry picked from commit 0f160ab)

.buildkite/ftr_security_stateful_configs.yml

@@ -100,4 +100,5 @@ enabled:
   - x-pack/test/cloud_security_posture_functional/config.ts
   - x-pack/test/cloud_security_posture_functional/config.agentless.ts
   - x-pack/test/cloud_security_posture_functional/data_views/config.ts
-  - x-pack/test/automatic_import_api_integration/security/config_basic.ts
+  - x-pack/test/automatic_import_api_integration/apis/config_basic.ts
+  - x-pack/test/automatic_import_api_integration/apis/config_graphs.ts

x-pack/test/automatic_import_api_integration/security/config_basic.ts → x-pack/test/automatic_import_api_integration/apis/config_basic.ts

@@ -7,8 +7,7 @@
 
 import { createTestConfig } from '../common/config';
 
-// eslint-disable-next-line import/no-default-export
-export default createTestConfig('security', {
+export default createTestConfig('apis', {
   license: 'basic',
   ssl: true,
   testFiles: [require.resolve('./tests/basic')],

x-pack/test/automatic_import_api_integration/apis/config_graphs.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createTestConfig } from '../common/config';
+
+export default createTestConfig('apis', {
+  license: 'trial',
+  ssl: true,
+  testFiles: [require.resolve('./tests/graphs')],
+  publicBaseUrl: true,
+});

x-pack/test/automatic_import_api_integration/security/tests/basic/graphs/analyze_logs.ts → x-pack/test/automatic_import_api_integration/apis/tests/basic/graphs/analyze_logs.ts

@@ -9,7 +9,6 @@ import { postAnalyzeLogs } from '../../../../common/lib/api/analyze_logs';
 import { FtrProviderContext } from '../../../../../common/ftr_provider_context';
 import { User } from '../../../../common/lib/authentication/types';
 
-// eslint-disable-next-line import/no-default-export
 export default ({ getService }: FtrProviderContext): void => {
   const supertest = getService('supertest');
 

x-pack/test/automatic_import_api_integration/security/tests/basic/graphs/categorization.ts → x-pack/test/automatic_import_api_integration/apis/tests/basic/graphs/categorization.ts

@@ -9,7 +9,6 @@ import { postCategorization } from '../../../../common/lib/api/categorization';
 import { FtrProviderContext } from '../../../../../common/ftr_provider_context';
 import { User } from '../../../../common/lib/authentication/types';
 
-// eslint-disable-next-line import/no-default-export
 export default ({ getService }: FtrProviderContext): void => {
   const supertest = getService('supertest');
 

x-pack/test/automatic_import_api_integration/security/tests/basic/graphs/ecs.ts → x-pack/test/automatic_import_api_integration/apis/tests/basic/graphs/ecs.ts

@@ -9,7 +9,6 @@ import { postEcsMapping } from '../../../../common/lib/api/ecs';
 import { FtrProviderContext } from '../../../../../common/ftr_provider_context';
 import { User } from '../../../../common/lib/authentication/types';
 
-// eslint-disable-next-line import/no-default-export
 export default ({ getService }: FtrProviderContext): void => {
   const supertest = getService('supertest');
 
@@ -26,6 +25,7 @@ export default ({ getService }: FtrProviderContext): void => {
           },
           connectorId: 'bedrock-connector',
         },
+        expectedHttpCode: 404,
         auth: {
           user: { username: 'elastic', password: 'elastic' } as User,
         },

x-pack/test/automatic_import_api_integration/security/tests/basic/graphs/related.ts → x-pack/test/automatic_import_api_integration/apis/tests/basic/graphs/related.ts

@@ -9,7 +9,6 @@ import { postRelated } from '../../../../common/lib/api/related';
 import { FtrProviderContext } from '../../../../../common/ftr_provider_context';
 import { User } from '../../../../common/lib/authentication/types';
 
-// eslint-disable-next-line import/no-default-export
 export default ({ getService }: FtrProviderContext): void => {
   const supertest = getService('supertest');
 

x-pack/test/automatic_import_api_integration/security/tests/basic/index.ts → x-pack/test/automatic_import_api_integration/apis/tests/basic/index.ts

@@ -12,7 +12,6 @@ import {
   activateUserProfiles,
 } from '../../../common/lib/authentication';
 
-// eslint-disable-next-line import/no-default-export
 export default ({ loadTestFile, getService }: FtrProviderContext): void => {
   describe('Automatic Import enabled: basic', function () {
     before(async () => {

x-pack/test/automatic_import_api_integration/apis/tests/graphs/categorization.ts

@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../../../../common/ftr_provider_context';
+import { postCategorization } from '../../../common/lib/api/categorization';
+import { User } from '../../../common/lib/authentication/types';
+import { BadRequestError } from '../../../common/lib/error/error';
+
+export default function (providerContext: FtrProviderContext) {
+  const { getService } = providerContext;
+  const supertest = getService('supertest');
+  describe('Run categorization', () => {
+    it('should get 400 when trying to run categorization with invalid json', async () => {
+      const response = await postCategorization({
+        supertest,
+        req: {
+          packageName: 'some-package',
+          dataStreamName: 'some-data-stream',
+          rawSamples: ['{test:json}'],
+          samplesFormat: {
+            name: 'json',
+          },
+          connectorId: 'preconfigured-bedrock',
+          currentPipeline: { processors: [] },
+        },
+        expectedHttpCode: 400,
+        auth: {
+          user: { username: 'elastic', password: 'elastic' } as User,
+        },
+      });
+      if (response instanceof BadRequestError) {
+        expect(response.message).to.be("Expected property name or '}' in JSON at position 1");
+      } else {
+        expect().fail('Expected BadRequestError');
+      }
+    });
+    it('should get 400 when trying to run categorization without connector action', async () => {
+      const response = await postCategorization({
+        supertest,
+        req: {
+          packageName: 'some-package',
+          dataStreamName: 'some-data-stream',
+          rawSamples: ['{test:json}'],
+          samplesFormat: {
+            name: 'json',
+          },
+          connectorId: 'preconfigured-dummy',
+          currentPipeline: { processors: [] },
+        },
+        expectedHttpCode: 400,
+        auth: {
+          user: { username: 'elastic', password: 'elastic' } as User,
+        },
+      });
+      if (response instanceof BadRequestError) {
+        expect(response.message).to.be('Saved object [action/preconfigured-dummy] not found');
+      } else {
+        expect().fail('Expected BadRequestError');
+      }
+    });
+  });
+}

x-pack/test/automatic_import_api_integration/apis/tests/graphs/ecs_mapping.ts

@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../../../../common/ftr_provider_context';
+import { postEcsMapping } from '../../../common/lib/api/ecs';
+import { User } from '../../../common/lib/authentication/types';
+import { BadRequestError } from '../../../common/lib/error/error';
+
+export default function (providerContext: FtrProviderContext) {
+  const { getService } = providerContext;
+  const supertest = getService('supertest');
+  describe('Run ecs_mapping', () => {
+    it('should get 400 when trying to run ecs_mapping with invalid json', async () => {
+      const response = await postEcsMapping({
+        supertest,
+        req: {
+          packageName: 'some-package',
+          dataStreamName: 'some-data-stream',
+          rawSamples: ['{test:json}'],
+          samplesFormat: {
+            name: 'json',
+          },
+          connectorId: 'preconfigured-bedrock',
+        },
+        expectedHttpCode: 400,
+        auth: {
+          user: { username: 'elastic', password: 'elastic' } as User,
+        },
+      });
+      if (response instanceof BadRequestError) {
+        expect(response.message).to.be("Expected property name or '}' in JSON at position 1");
+      } else {
+        expect().fail('Expected BadRequestError');
+      }
+    });
+    it('should get 400 when trying to run ecs_mapping without connector action', async () => {
+      const response = await postEcsMapping({
+        supertest,
+        req: {
+          packageName: 'some-package',
+          dataStreamName: 'some-data-stream',
+          rawSamples: ['{test:json}'],
+          samplesFormat: {
+            name: 'json',
+          },
+          connectorId: 'preconfigured-dummy',
+        },
+        expectedHttpCode: 400,
+        auth: {
+          user: { username: 'elastic', password: 'elastic' } as User,
+        },
+      });
+      if (response instanceof BadRequestError) {
+        expect(response.message).to.be('Saved object [action/preconfigured-dummy] not found');
+      } else {
+        expect().fail('Expected BadRequestError');
+      }
+    });
+  });
+}

x-pack/test/automatic_import_api_integration/apis/tests/graphs/index.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../../../common/ftr_provider_context';
+import {
+  createUsersAndRoles,
+  deleteUsersAndRoles,
+  activateUserProfiles,
+} from '../../../common/lib/authentication';
+
+export default ({ loadTestFile, getService }: FtrProviderContext): void => {
+  describe('Automatic Import enabled: basic', function () {
+    before(async () => {
+      await createUsersAndRoles(getService);
+      // once a user profile is created the only way to remove it is to delete the user and roles, so best to activate
+      // before all the tests
+      await activateUserProfiles(getService);
+    });
+
+    after(async () => {
+      await deleteUsersAndRoles(getService);
+    });
+
+    // Basic
+    loadTestFile(require.resolve('./ecs_mapping'));
+    loadTestFile(require.resolve('./categorization'));
+    loadTestFile(require.resolve('./related'));
+  });
+};

x-pack/test/automatic_import_api_integration/apis/tests/graphs/related.ts

@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../../../../common/ftr_provider_context';
+import { postRelated } from '../../../common/lib/api/related';
+import { User } from '../../../common/lib/authentication/types';
+import { BadRequestError } from '../../../common/lib/error/error';
+
+export default function (providerContext: FtrProviderContext) {
+  const { getService } = providerContext;
+  const supertest = getService('supertest');
+  describe('Run related', () => {
+    it('should get 400 when trying to run related with invalid json', async () => {
+      const response = await postRelated({
+        supertest,
+        req: {
+          packageName: 'some-package',
+          dataStreamName: 'some-data-stream',
+          rawSamples: ['{test:json}'],
+          samplesFormat: {
+            name: 'json',
+          },
+          connectorId: 'preconfigured-bedrock',
+          currentPipeline: { processors: [] },
+        },
+        expectedHttpCode: 400,
+        auth: {
+          user: { username: 'elastic', password: 'elastic' } as User,
+        },
+      });
+      if (response instanceof BadRequestError) {
+        expect(response.message).to.be("Expected property name or '}' in JSON at position 1");
+      } else {
+        expect().fail('Expected BadRequestError');
+      }
+    });
+    it('should get 400 when trying to run related without connector action', async () => {
+      const response = await postRelated({
+        supertest,
+        req: {
+          packageName: 'some-package',
+          dataStreamName: 'some-data-stream',
+          rawSamples: ['{test:json}'],
+          samplesFormat: {
+            name: 'json',
+          },
+          connectorId: 'preconfigured-dummy',
+          currentPipeline: { processors: [] },
+        },
+        expectedHttpCode: 400,
+        auth: {
+          user: { username: 'elastic', password: 'elastic' } as User,
+        },
+      });
+      if (response instanceof BadRequestError) {
+        expect(response.message).to.be('Saved object [action/preconfigured-dummy] not found');
+      } else {
+        expect().fail('Expected BadRequestError');
+      }
+    });
+  });
+}

x-pack/test/automatic_import_api_integration/common/lib/api/categorization.ts

@@ -12,6 +12,7 @@ import {
 } from '@kbn/integration-assistant-plugin/common';
 import { superUser } from '../authentication/users';
 import { User } from '../authentication/types';
+import { BadRequestError } from '../error/error';
 
 export const postCategorization = async ({
   supertest,
@@ -23,7 +24,7 @@ export const postCategorization = async ({
   req: CategorizationRequestBody;
   expectedHttpCode?: number;
   auth: { user: User };
-}): Promise<CategorizationResponse> => {
+}): Promise<CategorizationResponse | BadRequestError> => {
   const { body: response } = await supertest
     .post(`${CATEGORIZATION_GRAPH_PATH}`)
     .send(req)
@@ -32,5 +33,9 @@ export const postCategorization = async ({
     .auth(auth.user.username, auth.user.password)
     .expect(expectedHttpCode);
 
+  if (response.statusCode === 400) {
+    return new BadRequestError(response.message);
+  }
+
   return response;
 };

x-pack/test/automatic_import_api_integration/common/lib/api/ecs.ts

@@ -12,6 +12,7 @@ import {
 } from '@kbn/integration-assistant-plugin/common';
 import { superUser } from '../authentication/users';
 import { User } from '../authentication/types';
+import { BadRequestError } from '../error/error';
 
 export const postEcsMapping = async ({
   supertest,
@@ -23,7 +24,7 @@ export const postEcsMapping = async ({
   req: EcsMappingRequestBody;
   expectedHttpCode?: number;
   auth: { user: User };
-}): Promise<EcsMappingResponse> => {
+}): Promise<EcsMappingResponse | BadRequestError> => {
   const { body: response } = await supertest
     .post(`${ECS_GRAPH_PATH}`)
     .send(req)
@@ -32,5 +33,9 @@ export const postEcsMapping = async ({
     .auth(auth.user.username, auth.user.password)
     .expect(expectedHttpCode);
 
+  if (response.statusCode === 400) {
+    return new BadRequestError(response.message);
+  }
+
   return response;
 };

x-pack/test/automatic_import_api_integration/common/lib/api/related.ts

@@ -12,6 +12,7 @@ import {
 } from '@kbn/integration-assistant-plugin/common';
 import { superUser } from '../authentication/users';
 import { User } from '../authentication/types';
+import { BadRequestError } from '../error/error';
 
 export const postRelated = async ({
   supertest,
@@ -23,7 +24,7 @@ export const postRelated = async ({
   req: RelatedRequestBody;
   expectedHttpCode?: number;
   auth: { user: User };
-}): Promise<RelatedResponse> => {
+}): Promise<RelatedResponse | BadRequestError> => {
   const { body: response } = await supertest
     .post(`${RELATED_GRAPH_PATH}`)
     .send(req)
@@ -32,5 +33,9 @@ export const postRelated = async ({
     .auth(auth.user.username, auth.user.password)
     .expect(expectedHttpCode);
 
+  if (response.statusCode === 400) {
+    return new BadRequestError(response.message);
+  }
+
   return response;
 };