Merge branch 'main' into feature-fleet-space-filtering

elastic · Jun 12, 2024 · adfe22c · adfe22c
2 parents 967a448 + 7250f54
commit adfe22c
Show file tree

Hide file tree

Showing 40 changed files with 1,581 additions and 99 deletions.
diff --git a/.buildkite/scripts/packer_cache.sh b/.buildkite/scripts/packer_cache.sh
@@ -17,6 +17,3 @@ done
 for version in $(cat versions.json | jq -r '.versions[].version'); do
   node x-pack/plugins/security_solution/scripts/endpoint/agent_downloader --version "$version"
 done
-
-echo "--- Cloning repos for docs build"
-node scripts/validate_next_docs --clone-only
diff --git a/fleet_packages.json b/fleet_packages.json
@@ -56,6 +56,6 @@
   },
   {
     "name": "security_detection_engine",
-    "version": "8.14.2"
+    "version": "8.14.3"
   }
 ]
diff --git a/packages/kbn-apm-synthtrace-client/src/lib/infra/docker_container.ts b/packages/kbn-apm-synthtrace-client/src/lib/infra/docker_container.ts
@@ -13,14 +13,22 @@ import { Serializable } from '../serializable';
 interface DockerContainerDocument extends Fields {
   'container.id': string;
   'metricset.name'?: string;
+  'container.name'?: string;
+  'container.image.name'?: string;
+  'container.runtime'?: string;
+  'host.name'?: string;
+  'cloud.provider'?: string;
+  'cloud.instance.id'?: string;
+  'cloud.image.id'?: string;
+  'event.dataset'?: string;
 }
 
 export class DockerContainer extends Entity<DockerContainerDocument> {
   metrics() {
     return new DockerContainerMetrics({
       ...this.fields,
-      'docker.cpu.total.pct': 25,
-      'docker.memory.usage.pct': 20,
+      'docker.cpu.total.pct': 0.25,
+      'docker.memory.usage.pct': 0.2,
       'docker.network.inbound.bytes': 100,
       'docker.network.outbound.bytes': 200,
       'docker.diskio.read.ops': 10,

diff --git a/packages/kbn-apm-synthtrace-client/src/lib/infra/k8s_container.ts b/packages/kbn-apm-synthtrace-client/src/lib/infra/k8s_container.ts
@@ -15,6 +15,14 @@ interface K8sContainerDocument extends Fields {
   'kubernetes.pod.uid': string;
   'kubernetes.node.name': string;
   'metricset.name'?: string;
+  'container.name'?: string;
+  'container.image.name'?: string;
+  'container.runtime'?: string;
+  'host.name'?: string;
+  'cloud.provider'?: string;
+  'cloud.instance.id'?: string;
+  'cloud.image.id'?: string;
+  'event.dataset'?: string;
 }
 
 export class K8sContainer extends Entity<K8sContainerDocument> {

diff --git a/packages/kbn-apm-synthtrace/src/scenarios/infra_docker_containers.ts b/packages/kbn-apm-synthtrace/src/scenarios/infra_docker_containers.ts
@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-import { InfraDocument, infra } from '@kbn/apm-synthtrace-client';
+import { InfraDocument, infra, generateShortId } from '@kbn/apm-synthtrace-client';
 
 import { Scenario } from '../cli/scenario';
 import { withClient } from '../lib/utils/with_client';
@@ -19,7 +19,20 @@ const scenario: Scenario<InfraDocument> = async (runOptions) => {
 
       const CONTAINERS = Array(numContainers)
         .fill(0)
-        .map((_, idx) => infra.dockerContainer(`container-${idx}`));
+        .map((_, idx) => {
+          const id = generateShortId();
+          return infra.dockerContainer(id).defaults({
+            'container.name': `container-${idx}`,
+            'container.id': id,
+            'container.runtime': 'docker',
+            'container.image.name': 'image-1',
+            'host.name': 'host-1',
+            'cloud.instance.id': 'instance-1',
+            'cloud.image.id': 'image-1',
+            'cloud.provider': 'aws',
+            'event.dataset': 'docker.container',
+          });
+        });
 
       const containers = range
         .interval('30s')

diff --git a/packages/kbn-apm-synthtrace/src/scenarios/infra_k8s_containers.ts b/packages/kbn-apm-synthtrace/src/scenarios/infra_k8s_containers.ts
@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-import { InfraDocument, infra } from '@kbn/apm-synthtrace-client';
+import { InfraDocument, infra, generateShortId } from '@kbn/apm-synthtrace-client';
 
 import { Scenario } from '../cli/scenario';
 import { withClient } from '../lib/utils/with_client';
@@ -19,7 +19,22 @@ const scenario: Scenario<InfraDocument> = async (runOptions) => {
 
       const CONTAINERS = Array(numContainers)
         .fill(0)
-        .map((_, idx) => infra.k8sContainer(`container-${idx}`, `pod-${idx}`, `node-${idx}`));
+        .map((_, idx) => {
+          const id = generateShortId();
+          return infra.k8sContainer(id, `pod-${idx}`, `node-${idx}`).defaults({
+            'container.id': id,
+            'kubernetes.pod.uid': `pod-${idx}`,
+            'kubernetes.node.name': `node-${idx}`,
+            'container.name': `container-${idx}`,
+            'container.runtime': 'docker',
+            'container.image.name': 'image-1',
+            'host.name': 'host-1',
+            'cloud.instance.id': 'instance-1',
+            'cloud.image.id': 'image-1',
+            'cloud.provider': 'aws',
+            'event.dataset': 'kubernetes.container',
+          });
+        });
 
       const containers = range
         .interval('30s')

diff --git a/packages/kbn-discover-utils/src/utils/build_data_record.test.ts b/packages/kbn-discover-utils/src/utils/build_data_record.test.ts
@@ -22,7 +22,10 @@ describe('Data table record utils', () => {
 
   describe('buildDataTableRecordList', () => {
     test('should return a list of DataTableRecord', () => {
-      const result = buildDataTableRecordList(esHitsMock, dataViewMock);
+      const result = buildDataTableRecordList({
+        records: esHitsMock,
+        dataView: dataViewMock,
+      });
       result.forEach((doc) => {
         expect(doc).toHaveProperty('id');
         expect(doc).toHaveProperty('raw');
@@ -32,7 +35,9 @@ describe('Data table record utils', () => {
     });
 
     test('should support processing each record', () => {
-      const result = buildDataTableRecordList(esHitsMock, dataViewMock, {
+      const result = buildDataTableRecordList({
+        records: esHitsMock,
+        dataView: dataViewMock,
         processRecord: (record) => ({ ...record, id: 'custom-id' }),
       });
       result.forEach((doc) => {

diff --git a/packages/kbn-discover-utils/src/utils/build_data_record.ts b/packages/kbn-discover-utils/src/utils/build_data_record.ts
@@ -32,15 +32,19 @@ export function buildDataTableRecord(
 
 /**
  * Helper to build multiple DataTableRecords at once, saved a bit of testing code lines
- * @param docs Array of documents returned from Elasticsearch
+ * @param records Array of documents returned from Elasticsearch
  * @param dataView this current data view
  */
-export function buildDataTableRecordList<T extends DataTableRecord = DataTableRecord>(
-  docs: EsHitRecord[],
-  dataView?: DataView,
-  { processRecord }: { processRecord?: (record: DataTableRecord) => T } = {}
-): DataTableRecord[] {
-  return docs.map((doc) => {
+export function buildDataTableRecordList<T extends DataTableRecord = DataTableRecord>({
+  records,
+  dataView,
+  processRecord,
+}: {
+  records: EsHitRecord[];
+  dataView?: DataView;
+  processRecord?: (record: DataTableRecord) => T;
+}): DataTableRecord[] {
+  return records.map((doc) => {
     const record = buildDataTableRecord(doc, dataView);
     return processRecord ? processRecord(record) : record;
   });

diff --git a/packages/kbn-search-connectors/types/native_connectors.ts b/packages/kbn-search-connectors/types/native_connectors.ts
@@ -886,8 +886,56 @@ export const NATIVE_CONNECTOR_DEFINITIONS: Record<string, NativeConnector | unde
         validations: [],
         value: false,
       },
+      use_document_level_security: {
+        default_value: null,
+        depends_on: [],
+        display: DisplayType.TOGGLE,
+        label: ENABLE_DOCUMENT_LEVEL_SECURITY_LABEL,
+        options: [],
+        order: 8,
+        required: true,
+        sensitive: false,
+        tooltip: getEnableDocumentLevelSecurityTooltip(
+          i18n.translate('searchConnectors.nativeConnectors.dropbox.tooltipName', {
+            defaultMessage: 'Dropbox',
+          })
+        ),
+        type: FieldType.BOOLEAN,
+        ui_restrictions: [],
+        validations: [],
+        value: false,
+      },
+      include_inherited_users_and_groups: {
+        default_value: null,
+        depends_on: [{ field: 'use_document_level_security', value: true }],
+        display: DisplayType.TOGGLE,
+        label: i18n.translate(
+          'searchConnectors.nativeConnectors.dropbox.includeInheritedUsersAndGroups.label',
+          {
+            defaultMessage: 'Include groups and inherited users',
+          }
+        ),
+        options: [],
+        order: 9,
+        required: false,
+        sensitive: false,
+        tooltip: i18n.translate(
+          'searchConnectors.nativeConnectors.dropbox.includeInheritedUsersAndGroups.tooltip',
+          {
+            defaultMessage:
+              'Include groups and inherited users when indexing permissions. Enabling this configurable field will cause a significant performance degradation.',
+          }
+        ),
+        type: FieldType.BOOLEAN,
+        ui_restrictions: [],
+        validations: [],
+        value: false,
+      },
     },
     features: {
+      [FeatureName.DOCUMENT_LEVEL_SECURITY]: {
+        enabled: true,
+      },
       [FeatureName.SYNC_RULES]: {
         advanced: { enabled: true },
         basic: { enabled: true },

diff --git a/src/plugins/discover/public/application/context/services/context.predecessors.test.ts b/src/plugins/discover/public/application/context/services/context.predecessors.test.ts
@@ -104,7 +104,10 @@ describe('context predecessors', function () {
         ({ rows }) => {
           expect(mockSearchSource.fetch$.calledOnce).toBe(true);
           expect(rows).toEqual(
-            buildDataTableRecordList(mockSearchSource._stubHits.slice(0, 3), dataView)
+            buildDataTableRecordList({
+              records: mockSearchSource._stubHits.slice(0, 3),
+              dataView,
+            })
           );
         }
       );
@@ -136,7 +139,10 @@ describe('context predecessors', function () {
           expect(Object.keys(last(intervals) ?? {})).toEqual(['format', 'gte']);
           expect(intervals.length).toBeGreaterThan(1);
           expect(rows).toEqual(
-            buildDataTableRecordList(mockSearchSource._stubHits.slice(0, 3), dataView)
+            buildDataTableRecordList({
+              records: mockSearchSource._stubHits.slice(0, 3),
+              dataView,
+            })
           );
         }
       );
@@ -172,7 +178,10 @@ describe('context predecessors', function () {
           expect(intervals.length).toBeGreaterThan(1);
 
           expect(rows).toEqual(
-            buildDataTableRecordList(mockSearchSource._stubHits.slice(-3), dataView)
+            buildDataTableRecordList({
+              records: mockSearchSource._stubHits.slice(-3),
+              dataView,
+            })
           );
         }
       );
@@ -263,7 +272,10 @@ describe('context predecessors', function () {
           expect(removeFieldsSpy.calledOnce).toBe(true);
           expect(setFieldsSpy.calledOnce).toBe(true);
           expect(rows).toEqual(
-            buildDataTableRecordList(mockSearchSource._stubHits.slice(0, 3), dataView)
+            buildDataTableRecordList({
+              records: mockSearchSource._stubHits.slice(0, 3),
+              dataView,
+            })
           );
         }
       );

diff --git a/src/plugins/discover/public/application/context/services/context.successors.test.ts b/src/plugins/discover/public/application/context/services/context.successors.test.ts
@@ -104,7 +104,10 @@ describe('context successors', function () {
         ({ rows }) => {
           expect(mockSearchSource.fetch$.calledOnce).toBe(true);
           expect(rows).toEqual(
-            buildDataTableRecordList(mockSearchSource._stubHits.slice(-3), dataView)
+            buildDataTableRecordList({
+              records: mockSearchSource._stubHits.slice(-3),
+              dataView,
+            })
           );
         }
       );
@@ -136,7 +139,10 @@ describe('context successors', function () {
           expect(Object.keys(last(intervals) ?? {})).toEqual(['format', 'lte']);
           expect(intervals.length).toBeGreaterThan(1);
           expect(rows).toEqual(
-            buildDataTableRecordList(mockSearchSource._stubHits.slice(-3), dataView)
+            buildDataTableRecordList({
+              records: mockSearchSource._stubHits.slice(-3),
+              dataView,
+            })
           );
         }
       );
@@ -166,7 +172,10 @@ describe('context successors', function () {
           expect(moment(last(intervals)?.gte).valueOf()).toBeGreaterThan(MS_PER_DAY * 2200);
           expect(intervals.length).toBeGreaterThan(1);
           expect(rows).toEqual(
-            buildDataTableRecordList(mockSearchSource._stubHits.slice(0, 4), dataView)
+            buildDataTableRecordList({
+              records: mockSearchSource._stubHits.slice(0, 4),
+              dataView,
+            })
           );
         }
       );
@@ -252,7 +261,10 @@ describe('context successors', function () {
         ({ rows, interceptedWarnings }) => {
           expect(mockSearchSource.fetch$.calledOnce).toBe(true);
           expect(rows).toEqual(
-            buildDataTableRecordList(mockSearchSource._stubHits.slice(-3), dataView)
+            buildDataTableRecordList({
+              records: mockSearchSource._stubHits.slice(-3),
+              dataView,
+            })
           );
           const setFieldsSpy = mockSearchSource.setField.withArgs('fields');
           const removeFieldsSpy = mockSearchSource.removeField.withArgs('fieldsFromSource');
@@ -324,7 +336,10 @@ describe('context successors', function () {
         ({ rows, interceptedWarnings }) => {
           expect(mockSearchSource.fetch$.calledOnce).toBe(true);
           expect(rows).toEqual(
-            buildDataTableRecordList(mockSearchSource._stubHits.slice(-3), dataView)
+            buildDataTableRecordList({
+              records: mockSearchSource._stubHits.slice(-3),
+              dataView,
+            })
           );
           expect(dataPluginMock.search.showWarnings).toHaveBeenCalledTimes(1);
           expect(interceptedWarnings?.length).toBe(1);

diff --git a/src/plugins/discover/public/application/main/data_fetching/fetch_documents.ts b/src/plugins/discover/public/application/main/data_fetching/fetch_documents.ts
@@ -10,7 +10,6 @@ import { filter, map } from 'rxjs';
 import { lastValueFrom } from 'rxjs';
 import { isRunningResponse, ISearchSource } from '@kbn/data-plugin/public';
 import { buildDataTableRecordList } from '@kbn/discover-utils';
-import type { EsHitRecord } from '@kbn/discover-utils/types';
 import type { SearchResponseWarning } from '@kbn/search-response-warnings';
 import { DataViewType } from '@kbn/data-views-plugin/public';
 import type { RecordsFetchResponse } from '../../types';
@@ -67,7 +66,9 @@ export const fetchDocuments = (
     .pipe(
       filter((res) => !isRunningResponse(res)),
       map((res) => {
-        return buildDataTableRecordList(res.rawResponse.hits.hits as EsHitRecord[], dataView, {
+        return buildDataTableRecordList({
+          records: res.rawResponse.hits.hits,
+          dataView,
           processRecord: (record) => services.profilesManager.resolveDocumentProfile({ record }),
         });
       })

diff --git a/src/plugins/navigation/kibana.jsonc b/src/plugins/navigation/kibana.jsonc
@@ -6,7 +6,7 @@
     "id": "navigation",
     "server": true,
     "browser": true,
-    "optionalPlugins": ["cloud", "cloudExperiments"],
+    "optionalPlugins": ["cloud", "cloudExperiments", "spaces"],
     "requiredPlugins": ["unifiedSearch"],
     "requiredBundles": []
   }