Merge branch 'main' into security/rules/4362-advanced-preview

.buildkite/ftr_configs.yml

@@ -32,6 +32,7 @@ disabled:
   - x-pack/test/security_solution_cypress/response_ops_cli_config.ts
   - x-pack/test/security_solution_cypress/upgrade_config.ts
   - x-pack/test/security_solution_cypress/visual_config.ts
+  - x-pack/test/threat_intelligence_cypress/visual_config.ts
   - x-pack/test/functional_enterprise_search/with_host_configured.config.ts
   - x-pack/plugins/apm/ftr_e2e/ftr_config_open.ts
   - x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts

.buildkite/scripts/steps/es_snapshots/build.sh

@@ -14,10 +14,6 @@ mkdir -p "$destination"
 mkdir -p elasticsearch && cd elasticsearch
 
 export ELASTICSEARCH_BRANCH="${ELASTICSEARCH_BRANCH:-$BUILDKITE_BRANCH}"
-# Until ES renames their master branch to main...
-if [[ "$ELASTICSEARCH_BRANCH" == "main" ]]; then
-  export ELASTICSEARCH_BRANCH="master"
-fi
 
 if [[ ! -d .git ]]; then
   git init

.github/CODEOWNERS

@@ -589,6 +589,12 @@ x-pack/plugins/session_view @elastic/awp-platform
 x-pack/plugins/security_solution/public/common/components/sessions_viewer @elastic/awp-platform
 x-pack/plugins/security_solution/public/kubernetes @elastic/awp-platform
 
+## Security Solution sub teams - Protections Experience
+x-pack/plugins/threat_intelligence @elastic/protections-experience
+x-pack/plugins/security_solution/public/threat_intelligence @elastic/protections-experience
+x-pack/test/threat_intelligence_cypress @elastic/protections-experience
+
+
 # Security Intelligence And Analytics
 /x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules @elastic/security-intelligence-analytics
 

docs/developer/plugin-list.asciidoc

@@ -639,6 +639,10 @@ Documentation: https://www.elastic.co/guide/en/kibana/master/task-manager-produc
 |Gathers all usage collection, retrieving them from both: OSS and X-Pack plugins.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/threat_intelligence/README.md[threatIntelligence]
+|Elastic Threat Intelligence makes it easy to analyze and investigate potential security threats by aggregating data from multiple sources in one place. You’ll be able to view data from all activated threat intelligence feeds and take action.
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/timelines/README.md[timelines]
 |Timelines is a plugin that provides a grid component with accompanying server side apis to help users identify events of interest and perform root cause analysis within Kibana.
 

packages/analytics/shippers/elastic_v3/server/src/server_shipper.test.ts

@@ -47,7 +47,7 @@ describe('ElasticV3ServerShipper', () => {
       initContext
     );
     // eslint-disable-next-line dot-notation
-    shipper['firstTimeOffline'] = null;
+    shipper['firstTimeOffline'] = null; // The tests think connectivity is OK initially for easier testing.
   });
 
   afterEach(() => {
@@ -57,7 +57,7 @@ describe('ElasticV3ServerShipper', () => {
 
   test('set optIn should update the isOptedIn$ observable', () => {
     // eslint-disable-next-line dot-notation
-    const getInternalOptIn = () => shipper['isOptedIn'];
+    const getInternalOptIn = () => shipper['isOptedIn$'].value;
 
     // Initially undefined
     expect(getInternalOptIn()).toBeUndefined();
@@ -342,97 +342,242 @@ describe('ElasticV3ServerShipper', () => {
     })
   );
 
-  test(
-    'connectivity check is run after report failure',
-    fakeSchedulers(async (advance) => {
-      fetchMock.mockRejectedValueOnce(new Error('Failed to fetch'));
-      shipper.reportEvents(events);
-      shipper.optIn(true);
-      const counter = firstValueFrom(shipper.telemetryCounter$);
-      setLastBatchSent(Date.now() - 10 * MINUTES);
-      advance(10 * MINUTES);
-      expect(fetchMock).toHaveBeenNthCalledWith(
-        1,
-        'https://telemetry-staging.elastic.co/v3/send/test-channel',
-        {
-          body: '{"timestamp":"2020-01-01T00:00:00.000Z","event_type":"test-event-type","context":{},"properties":{}}\n',
-          headers: {
-            'content-type': 'application/x-ndjson',
-            'x-elastic-cluster-id': 'UNKNOWN',
-            'x-elastic-stack-version': '1.2.3',
-          },
-          method: 'POST',
-          query: { debug: true },
-        }
+  describe('Connectivity Checks', () => {
+    describe('connectivity check when connectivity is confirmed (firstTimeOffline === null)', () => {
+      test.each([undefined, false, true])('does not run for opt-in %p', (optInValue) =>
+        fakeSchedulers(async (advance) => {
+          if (optInValue !== undefined) {
+            shipper.optIn(optInValue);
+          }
+
+          // From the start, it doesn't check connectivity because already confirmed
+          expect(fetchMock).not.toHaveBeenCalledWith(
+            'https://telemetry-staging.elastic.co/v3/send/test-channel',
+            { method: 'OPTIONS' }
+          );
+
+          // Wait a big time (1 minute should be enough, but for the sake of tests...)
+          advance(10 * MINUTES);
+          await nextTick();
+
+          expect(fetchMock).not.toHaveBeenCalledWith(
+            'https://telemetry-staging.elastic.co/v3/send/test-channel',
+            { method: 'OPTIONS' }
+          );
+        })()
       );
-      await expect(counter).resolves.toMatchInlineSnapshot(`
-        Object {
-          "code": "Failed to fetch",
-          "count": 1,
-          "event_type": "test-event-type",
-          "source": "elastic_v3_server",
-          "type": "failed",
-        }
-      `);
-      fetchMock.mockRejectedValueOnce(new Error('Failed to fetch'));
-      advance(1 * MINUTES);
-      await nextTick();
-      expect(fetchMock).toHaveBeenNthCalledWith(
-        2,
-        'https://telemetry-staging.elastic.co/v3/send/test-channel',
-        { method: 'OPTIONS' }
+    });
+
+    describe('connectivity check with initial unknown state of the connectivity', () => {
+      beforeEach(() => {
+        // eslint-disable-next-line dot-notation
+        shipper['firstTimeOffline'] = undefined; // Initial unknown state of the connectivity
+      });
+
+      test.each([undefined, false])('does not run for opt-in %p', (optInValue) =>
+        fakeSchedulers(async (advance) => {
+          if (optInValue !== undefined) {
+            shipper.optIn(optInValue);
+          }
+
+          // From the start, it doesn't check connectivity because already confirmed
+          expect(fetchMock).not.toHaveBeenCalledWith(
+            'https://telemetry-staging.elastic.co/v3/send/test-channel',
+            { method: 'OPTIONS' }
+          );
+
+          // Wait a big time (1 minute should be enough, but for the sake of tests...)
+          advance(10 * MINUTES);
+          await nextTick();
+
+          expect(fetchMock).not.toHaveBeenCalledWith(
+            'https://telemetry-staging.elastic.co/v3/send/test-channel',
+            { method: 'OPTIONS' }
+          );
+        })()
       );
-      fetchMock.mockResolvedValueOnce({ ok: false });
-      advance(2 * MINUTES);
-      await nextTick();
-      expect(fetchMock).toHaveBeenNthCalledWith(
-        3,
-        'https://telemetry-staging.elastic.co/v3/send/test-channel',
-        { method: 'OPTIONS' }
+
+      test('runs as soon as opt-in is set to true', () => {
+        shipper.optIn(true);
+
+        // From the start, it doesn't check connectivity because opt-in is not true
+        expect(fetchMock).toHaveBeenNthCalledWith(
+          1,
+          'https://telemetry-staging.elastic.co/v3/send/test-channel',
+          { method: 'OPTIONS' }
+        );
+      });
+    });
+
+    describe('connectivity check with the connectivity confirmed to be faulty', () => {
+      beforeEach(() => {
+        // eslint-disable-next-line dot-notation
+        shipper['firstTimeOffline'] = 100; // Failed at some point
+      });
+
+      test.each([undefined, false])('does not run for opt-in %p', (optInValue) =>
+        fakeSchedulers(async (advance) => {
+          if (optInValue !== undefined) {
+            shipper.optIn(optInValue);
+          }
+
+          // From the start, it doesn't check connectivity because already confirmed
+          expect(fetchMock).not.toHaveBeenCalledWith(
+            'https://telemetry-staging.elastic.co/v3/send/test-channel',
+            { method: 'OPTIONS' }
+          );
+
+          // Wait a big time (1 minute should be enough, but for the sake of tests...)
+          advance(10 * MINUTES);
+          await nextTick();
+
+          expect(fetchMock).not.toHaveBeenCalledWith(
+            'https://telemetry-staging.elastic.co/v3/send/test-channel',
+            { method: 'OPTIONS' }
+          );
+        })()
       );
 
-      // let's see the effect of after 24 hours:
-      shipper.reportEvents(events);
-      // eslint-disable-next-line dot-notation
-      expect(shipper['internalQueue'].length).toBe(1);
-      // eslint-disable-next-line dot-notation
-      shipper['firstTimeOffline'] = 100;
+      test('runs as soon as opt-in is set to true', () => {
+        shipper.optIn(true);
 
-      fetchMock.mockResolvedValueOnce({ ok: false });
-      advance(4 * MINUTES);
-      await nextTick();
-      expect(fetchMock).toHaveBeenNthCalledWith(
-        4,
-        'https://telemetry-staging.elastic.co/v3/send/test-channel',
-        { method: 'OPTIONS' }
+        // From the start, it doesn't check connectivity because opt-in is not true
+        expect(fetchMock).toHaveBeenNthCalledWith(
+          1,
+          'https://telemetry-staging.elastic.co/v3/send/test-channel',
+          { method: 'OPTIONS' }
+        );
+      });
+    });
+
+    describe('after report failure', () => {
+      // generate the report failure for each test
+      beforeEach(
+        fakeSchedulers(async (advance) => {
+          fetchMock.mockRejectedValueOnce(new Error('Failed to fetch'));
+          shipper.reportEvents(events);
+          shipper.optIn(true);
+          const counter = firstValueFrom(shipper.telemetryCounter$);
+          setLastBatchSent(Date.now() - 10 * MINUTES);
+          advance(10 * MINUTES);
+          expect(fetchMock).toHaveBeenNthCalledWith(
+            1,
+            'https://telemetry-staging.elastic.co/v3/send/test-channel',
+            {
+              body: '{"timestamp":"2020-01-01T00:00:00.000Z","event_type":"test-event-type","context":{},"properties":{}}\n',
+              headers: {
+                'content-type': 'application/x-ndjson',
+                'x-elastic-cluster-id': 'UNKNOWN',
+                'x-elastic-stack-version': '1.2.3',
+              },
+              method: 'POST',
+              query: { debug: true },
+            }
+          );
+          await expect(counter).resolves.toMatchInlineSnapshot(`
+          Object {
+            "code": "Failed to fetch",
+            "count": 1,
+            "event_type": "test-event-type",
+            "source": "elastic_v3_server",
+            "type": "failed",
+          }
+        `);
+        })
       );
-      // eslint-disable-next-line dot-notation
-      expect(shipper['internalQueue'].length).toBe(0);
 
-      // New events are not added to the queue because it's been offline for 24 hours.
-      shipper.reportEvents(events);
-      // eslint-disable-next-line dot-notation
-      expect(shipper['internalQueue'].length).toBe(0);
-
-      // Regains connection
-      fetchMock.mockResolvedValueOnce({ ok: true });
-      advance(8 * MINUTES);
-      await nextTick();
-      expect(fetchMock).toHaveBeenNthCalledWith(
-        5,
-        'https://telemetry-staging.elastic.co/v3/send/test-channel',
-        { method: 'OPTIONS' }
+      test(
+        'connectivity check runs periodically',
+        fakeSchedulers(async (advance) => {
+          fetchMock.mockRejectedValueOnce(new Error('Failed to fetch'));
+          advance(1 * MINUTES);
+          await nextTick();
+          expect(fetchMock).toHaveBeenNthCalledWith(
+            2,
+            'https://telemetry-staging.elastic.co/v3/send/test-channel',
+            { method: 'OPTIONS' }
+          );
+          fetchMock.mockResolvedValueOnce({ ok: false });
+          advance(2 * MINUTES);
+          await nextTick();
+          expect(fetchMock).toHaveBeenNthCalledWith(
+            3,
+            'https://telemetry-staging.elastic.co/v3/send/test-channel',
+            { method: 'OPTIONS' }
+          );
+        })
       );
-      // eslint-disable-next-line dot-notation
-      expect(shipper['firstTimeOffline']).toBe(null);
+    });
+
+    describe('after being offline for longer than 24h', () => {
+      beforeEach(() => {
+        shipper.optIn(true);
+        shipper.reportEvents(events);
+        // eslint-disable-next-line dot-notation
+        expect(shipper['internalQueue'].length).toBe(1);
+        // eslint-disable-next-line dot-notation
+        shipper['firstTimeOffline'] = 100;
+      });
 
-      advance(16 * MINUTES);
-      await nextTick();
-      expect(fetchMock).not.toHaveBeenNthCalledWith(
-        6,
-        'https://telemetry-staging.elastic.co/v3/send/test-channel',
-        { method: 'OPTIONS' }
+      test(
+        'the following connectivity check clears the queue',
+        fakeSchedulers(async (advance) => {
+          fetchMock.mockRejectedValueOnce(new Error('Failed to fetch'));
+          advance(1 * MINUTES);
+          await nextTick();
+          expect(fetchMock).toHaveBeenNthCalledWith(
+            1,
+            'https://telemetry-staging.elastic.co/v3/send/test-channel',
+            { method: 'OPTIONS' }
+          );
+          // eslint-disable-next-line dot-notation
+          expect(shipper['internalQueue'].length).toBe(0);
+        })
       );
-    })
-  );
+
+      test(
+        'new events are not added to the queue',
+        fakeSchedulers(async (advance) => {
+          fetchMock.mockRejectedValueOnce(new Error('Failed to fetch'));
+          advance(1 * MINUTES);
+          await nextTick();
+          expect(fetchMock).toHaveBeenNthCalledWith(
+            1,
+            'https://telemetry-staging.elastic.co/v3/send/test-channel',
+            { method: 'OPTIONS' }
+          );
+          // eslint-disable-next-line dot-notation
+          expect(shipper['internalQueue'].length).toBe(0);
+
+          shipper.reportEvents(events);
+          // eslint-disable-next-line dot-notation
+          expect(shipper['internalQueue'].length).toBe(0);
+        })
+      );
+
+      test(
+        'regains the connection',
+        fakeSchedulers(async (advance) => {
+          fetchMock.mockResolvedValueOnce({ ok: true });
+          advance(1 * MINUTES);
+          await nextTick();
+          expect(fetchMock).toHaveBeenNthCalledWith(
+            1,
+            'https://telemetry-staging.elastic.co/v3/send/test-channel',
+            { method: 'OPTIONS' }
+          );
+          // eslint-disable-next-line dot-notation
+          expect(shipper['firstTimeOffline']).toBe(null);
+
+          advance(10 * MINUTES);
+          await nextTick();
+          expect(fetchMock).not.toHaveBeenNthCalledWith(
+            2,
+            'https://telemetry-staging.elastic.co/v3/send/test-channel',
+            { method: 'OPTIONS' }
+          );
+        })
+      );
+    });
+  });
 });