[Security Solution][Alerts] refactor rule execution tests

[vitaliidm]

Currently, functional rule executions tests use mock documents that stored in json and ingested into indices

Random test looks something like this:

    it('should generate 1 alert for unique combination of terms, one of which is a number', async () => {
      const rule: NewTermsRuleCreateProps = {
        ...getCreateNewTermsRulesSchemaMock('rule-1', true),
        index: ['new_terms'],
        new_terms_fields: ['user.name', 'user.id'],
        from: '2020-10-19T05:00:04.000Z',
        history_window_start: '2020-10-13T05:00:04.000Z',
      };

      const { previewId } = await previewRule({ supertest, rule });
      const previewAlerts = await getPreviewAlerts({ es, previewId });

      expect(previewAlerts.length).eql(1);
      expect(previewAlerts[0]._source?.['kibana.alert.new_terms']).eql(['user-0', '1']);
    });

For this test to work, particular document should exist that satisfy multiple conditions:

• its timestamp should be greater than from value
• should have unique combination of user.name and user.id in comparison with history window items
• values 'user-0', '1' are hardcoded in mock and it's not easy to see why exactly these values are asserted
• values 'user-0', '1' should be present in historical window, but not in combinaton with each other
• should be only such document in the data-set.
• Another document could either generate a new alert. Or if it has the same field values, test combination might not be a unique anymore

Given that a number of tests could be quite big, it becomes difficult to maintain data set and add new tests that could possibly break existing

Proposal

Provision test indices with documents during test run, so it will be easy to see how many documents are present and which fields will be asserted.

Possible solution could be something like this:

 it('should generate 1 alert for unique combination of terms, one of which is a number', async () => {
      const index = ['new_terms'];
      const date = moment();

      // documents in historical window
      addDocument({
        timestamp: date.subtract(3, 'hours').toISOString(),
        index,
        fields: {
          'user.name': 'user-0',
          'user.id': 0,
        },
      });

      addDocument({
        timestamp: date.subtract(4, 'hours').toISOString(),
        index,
        fields: {
          'user.name': 'user-1',
          'user.id': 1,
        },
      });

      // document ot generate alert
      addDocument({
        timestamp: date.toISOString(),
        index,
        fields: {
          'user.name': 'user-0',
          'user.id': 1,
        },
      });

      const rule: NewTermsRuleCreateProps = {
        ...getCreateNewTermsRulesSchemaMock('rule-1', true),
        index,
        new_terms_fields: ['user.name', 'user.id'],
        from: date.subtract(2, 'hours').toISOString(),
        history_window_start: date.subtract(6, 'hours').toISOString(),
      };

      const { previewId } = await previewRule({ supertest, rule });
      const previewAlerts = await getPreviewAlerts({ es, previewId });

      expect(previewAlerts.length).eql(1);
      expect(previewAlerts[0]._source?.['kibana.alert.new_terms']).eql(['user-0', '1']);
    });

Where addDocument could be helper method that would add document to index
Depends on implementation or performance, there could be different ways to setup indices

• indices can be created in before/add hooks, and tests could target added items by some identifier
• indices could be wiped out before each run

[elasticmachine]

Pinging @elastic/security-solution (Team: SecuritySolution)

[elasticmachine]

Pinging @elastic/security-detections-response (Team:Detections and Resp)

[vitaliidm]

Example how test written in ticket can be formatted according to draft PR

Before

    it('should generate 1 alert for unique combination of terms, one of which is a number', async () => {
      const rule: NewTermsRuleCreateProps = {
        ...getCreateNewTermsRulesSchemaMock('rule-1', true),
        index: ['new_terms'],
        new_terms_fields: ['user.name', 'user.id'],
        from: '2020-10-19T05:00:04.000Z',
        history_window_start: '2020-10-13T05:00:04.000Z',
      };

      const { previewId } = await previewRule({ supertest, rule });
      const previewAlerts = await getPreviewAlerts({ es, previewId });

      expect(previewAlerts.length).eql(1);
      expect(previewAlerts[0]._source?.['kibana.alert.new_terms']).eql(['user-0', '1']);
    });

Aftert

Single test

    it('should generate 1 alert for unique combination of terms, one of which is a number', async () => {
      const historicalWindowStart = '2020-10-13T05:00:04.000Z';
      const ruleExecutionStart = '2020-10-19T05:00:04.000Z';
      const testId = 'test-1';

      // historical window documents
      const historicalDocuments = [
        { user: { name: 'user-0', id: 0 } },
        { user: { name: 'user-1', id: 1 } },
      ];

      await indexEnhancedDocuments({
        interval: [historicalWindowStart, ruleExecutionStart],
        id: testId,
        documents: historicalDocuments,
      });

      // rule execution documents
      const ruleExecutionDocuments = [{ user: { name: 'user-0', id: 1 } }];

      await indexEnhancedDocuments({
        id: testId,
        documents: ruleExecutionDocuments,
      });

      const rule: NewTermsRuleCreateProps = {
        ...getCreateNewTermsRulesSchemaMock('rule-1', true),
        index: ['new_terms_refactored'],
        new_terms_fields: ['user.name', 'user.id'],
        from:ruleExecutionStart,
        history_window_start: historicalWindowStart,
        query: `id: "${testId}"`,
      };

      const { previewId } = await previewRule({ supertest, rule });
      const previewAlerts = await getPreviewAlerts({ es, previewId });

      expect(previewAlerts.length).eql(1);
      expect(previewAlerts[0]._source?.['kibana.alert.new_terms']).eql(['user-0', '1']);
    });

Multiple tests

When multiple tests needed to be run, additional boilerplate can be reduced to something like this:

    it.only('should generate 1 alert for unique combination of terms, one of which is a number', async () => {
      const testId = 'test-2';
      const historicalDocuments = [
        { user: { name: 'user-0', id: 0 } },
        { user: { name: 'user-1', id: 1 } },
      ];
      const ruleExecutionDocuments = [{ user: { name: 'user-0', id: 1 } }];

      await newTermsTestExecutionSetup({
        historicalDocuments,
        ruleExecutionDocuments,
        testId,
      });

      const rule: NewTermsRuleCreateProps = {
        ...getCreateNewTermsRulesSchemaMock('rule-1', true),
        index: ['new_terms_refactored'],
        new_terms_fields: ['user.name', 'user.id'],
        from: ruleExecutionStart,
        history_window_start: historicalWindowStart,
        query: `id: "${testId}"`,
      };

      const { previewId } = await previewRule({ supertest, rule });
      const previewAlerts = await getPreviewAlerts({ es, previewId });

      expect(previewAlerts.length).eql(1);
      expect(previewAlerts[0]._source?.['kibana.alert.new_terms']).eql(['user-0', '1']);
    });

where newTermsTestExecutionSetup :

    const historicalWindowStart = '2020-10-13T05:00:04.000Z';
    const ruleExecutionStart = '2020-10-19T05:00:04.000Z';
    const newTermsTestExecutionSetup = async ({
      historicalDocuments,
      ruleExecutionDocuments,
      testId,
    }: {
      historicalDocuments: Array<Record<string, unknown>>;
      ruleExecutionDocuments: Array<Record<string, unknown>>;
      testId: string;
    }) => {
      await indexEnhancedDocuments({
        interval: [historicalWindowStart, ruleExecutionStart],
        id: testId,
        documents: historicalDocuments,
      });

      await indexEnhancedDocuments({
        id: testId,
        documents: ruleExecutionDocuments,
      });
    };

[vitaliidm]

further updates in tests can be done according to README after #151192 has been merged