[Security Solution] update endpoint list api to support united index (#…

…112758)

x-pack/plugins/fleet/common/services/agent_status.ts

@@ -8,7 +8,7 @@
 import { AGENT_POLLING_THRESHOLD_MS } from '../constants';
 import type { Agent, AgentStatus } from '../types';
 
-export function getAgentStatus(agent: Agent, now: number = Date.now()): AgentStatus {
+export function getAgentStatus(agent: Agent): AgentStatus {
   const { last_checkin: lastCheckIn } = agent;
 
   if (!agent.active) {
@@ -41,36 +41,42 @@ export function getAgentStatus(agent: Agent, now: number = Date.now()): AgentSta
   return 'online';
 }
 
-export function buildKueryForEnrollingAgents() {
-  return 'not (last_checkin:*)';
+export function buildKueryForEnrollingAgents(path: string = '') {
+  return `not (${path}last_checkin:*)`;
 }
 
-export function buildKueryForUnenrollingAgents() {
-  return 'unenrollment_started_at:*';
+export function buildKueryForUnenrollingAgents(path: string = '') {
+  return `${path}unenrollment_started_at:*`;
 }
 
-export function buildKueryForOnlineAgents() {
-  return `not (${buildKueryForOfflineAgents()}) AND not (${buildKueryForErrorAgents()}) AND not (${buildKueryForUpdatingAgents()})`;
+export function buildKueryForOnlineAgents(path: string = '') {
+  return `not (${buildKueryForOfflineAgents(path)}) AND not (${buildKueryForErrorAgents(
+    path
+  )}) AND not (${buildKueryForUpdatingAgents(path)})`;
 }
 
-export function buildKueryForErrorAgents() {
-  return `(last_checkin_status:error or last_checkin_status:degraded) AND not (${buildKueryForUpdatingAgents()})`;
+export function buildKueryForErrorAgents(path: string = '') {
+  return `(${path}last_checkin_status:error or ${path}last_checkin_status:degraded) AND not (${buildKueryForUpdatingAgents(
+    path
+  )})`;
 }
 
-export function buildKueryForOfflineAgents() {
-  return `last_checkin < now-${
+export function buildKueryForOfflineAgents(path: string = '') {
+  return `${path}last_checkin < now-${
     (4 * AGENT_POLLING_THRESHOLD_MS) / 1000
-  }s AND not (${buildKueryForErrorAgents()}) AND not ( ${buildKueryForUpdatingAgents()} )`;
+  }s AND not (${buildKueryForErrorAgents(path)}) AND not ( ${buildKueryForUpdatingAgents(path)} )`;
 }
 
-export function buildKueryForUpgradingAgents() {
-  return '(upgrade_started_at:*) and not (upgraded_at:*)';
+export function buildKueryForUpgradingAgents(path: string = '') {
+  return `(${path}upgrade_started_at:*) and not (${path}upgraded_at:*)`;
 }
 
-export function buildKueryForUpdatingAgents() {
-  return `(${buildKueryForUpgradingAgents()}) or (${buildKueryForEnrollingAgents()}) or (${buildKueryForUnenrollingAgents()})`;
+export function buildKueryForUpdatingAgents(path: string = '') {
+  return `(${buildKueryForUpgradingAgents(path)}) or (${buildKueryForEnrollingAgents(
+    path
+  )}) or (${buildKueryForUnenrollingAgents(path)})`;
 }
 
-export function buildKueryForInactiveAgents() {
-  return `active:false`;
+export function buildKueryForInactiveAgents(path: string = '') {
+  return `${path}active:false`;
 }

x-pack/plugins/security_solution/common/endpoint/constants.ts

@@ -20,6 +20,9 @@ export const metadataTransformPrefix = 'endpoint.metadata_current-default';
 /** The metadata Transform Name prefix with NO namespace and NO (package) version) */
 export const metadataTransformPattern = 'endpoint.metadata_current-*';
 
+export const METADATA_UNITED_TRANSFORM = 'endpoint.metadata_united-default';
+export const METADATA_UNITED_INDEX = '.metrics-endpoint.metadata_united_default';
+
 export const policyIndexPattern = 'metrics-endpoint.policy-*';
 export const telemetryIndexPattern = 'metrics-endpoint.telemetry-*';
 export const LIMITED_CONCURRENCY_ENDPOINT_ROUTE_TAG = 'endpoint:limited-concurrency';

x-pack/plugins/security_solution/common/endpoint/data_loaders/index_endpoint_hosts.ts

@@ -181,6 +181,14 @@ export async function indexEndpointHostDocs({
       await indexFleetActionsForHost(client, hostMetadata);
     }
 
+    hostMetadata = {
+      ...hostMetadata,
+      // since the united transform uses latest metadata transform as a source
+      // there is an extra delay and fleet-agents gets populated much sooner.
+      // we manually add a delay to the time sync field so that the united transform
+      // will pick up the latest metadata doc.
+      '@timestamp': hostMetadata['@timestamp'] + 60000,
+    };
     await client
       .index({
         index: metadataIndex,

x-pack/plugins/security_solution/common/endpoint/types/index.ts

@@ -6,7 +6,7 @@
  */
 
 import { ApplicationStart } from 'kibana/public';
-import { PackagePolicy, UpdatePackagePolicy } from '../../../../fleet/common';
+import { Agent, PackagePolicy, UpdatePackagePolicy } from '../../../../fleet/common';
 import { ManifestSchema } from '../schema/manifest';
 
 export * from './actions';
@@ -546,6 +546,16 @@ export type HostMetadata = Immutable<{
   data_stream: DataStream;
 }>;
 
+export type UnitedAgentMetadata = Immutable<{
+  agent: {
+    id: string;
+  };
+  united: {
+    endpoint: HostMetadata;
+    agent: Agent;
+  };
+}>;
+
 export interface LegacyEndpointEvent {
   '@timestamp': number;
   endgame: {

x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.test.ts

@@ -53,7 +53,7 @@ import {
 jest.mock('../../policy/store/services/ingest', () => ({
   sendGetAgentConfigList: () => Promise.resolve({ items: [] }),
   sendGetAgentPolicyList: () => Promise.resolve({ items: [] }),
-  sendGetEndpointSecurityPackage: () => Promise.resolve({}),
+  sendGetEndpointSecurityPackage: () => Promise.resolve({ version: '1.1.1' }),
   sendGetFleetAgentsWithEndpoint: () => Promise.resolve({ total: 0 }),
 }));
 

x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/store/middleware.ts

@@ -6,6 +6,8 @@
  */
 
 import { Dispatch } from 'redux';
+import semverGte from 'semver/functions/gte';
+
 import { CoreStart, HttpStart } from 'kibana/public';
 import {
   ActivityLog,
@@ -40,6 +42,7 @@ import {
   getMetadataTransformStats,
   isMetadataTransformStatsLoading,
   getActivityLogIsUninitializedOrHasSubsequentAPIError,
+  endpointPackageVersion,
 } from './selectors';
 import {
   AgentIdsPendingActions,
@@ -61,6 +64,7 @@ import {
   HOST_METADATA_LIST_ROUTE,
   BASE_POLICY_RESPONSE_ROUTE,
   metadataCurrentIndexPattern,
+  METADATA_UNITED_INDEX,
 } from '../../../../../common/endpoint/constants';
 import { IIndexPattern, Query } from '../../../../../../../../src/plugins/data/public';
 import {
@@ -85,13 +89,26 @@ export const endpointMiddlewareFactory: ImmutableMiddlewareFactory<EndpointState
   coreStart,
   depsStart
 ) => {
-  async function fetchIndexPatterns(): Promise<IIndexPattern[]> {
+  // this needs to be called after endpointPackageVersion is loaded (getEndpointPackageInfo)
+  // or else wrong pattern might be loaded
+  async function fetchIndexPatterns(
+    state: ImmutableObject<EndpointState>
+  ): Promise<IIndexPattern[]> {
+    const packageVersion = endpointPackageVersion(state) ?? '';
+    const parsedPackageVersion = packageVersion.includes('-')
+      ? packageVersion.substring(0, packageVersion.indexOf('-'))
+      : packageVersion;
+    const minUnitedIndexVersion = '1.2.0';
+    const indexPatternToFetch = semverGte(parsedPackageVersion, minUnitedIndexVersion)
+      ? METADATA_UNITED_INDEX
+      : metadataCurrentIndexPattern;
+
     const { indexPatterns } = depsStart.data;
     const fields = await indexPatterns.getFieldsForWildcard({
-      pattern: metadataCurrentIndexPattern,
+      pattern: indexPatternToFetch,
     });
     const indexPattern: IIndexPattern = {
-      title: metadataCurrentIndexPattern,
+      title: indexPatternToFetch,
       fields,
     };
     return [indexPattern];
@@ -379,7 +396,7 @@ async function endpointDetailsListMiddleware({
 }: {
   store: ImmutableMiddlewareAPI<EndpointState, AppAction>;
   coreStart: CoreStart;
-  fetchIndexPatterns: () => Promise<IIndexPattern[]>;
+  fetchIndexPatterns: (state: ImmutableObject<EndpointState>) => Promise<IIndexPattern[]>;
 }) {
   const { getState, dispatch } = store;
 
@@ -441,7 +458,7 @@ async function endpointDetailsListMiddleware({
   // get index pattern and fields for search bar
   if (patterns(getState()).length === 0) {
     try {
-      const indexPatterns = await fetchIndexPatterns();
+      const indexPatterns = await fetchIndexPatterns(getState());
       if (indexPatterns !== undefined) {
         dispatch({
           type: 'serverReturnedMetadataPatterns',

x-pack/plugins/security_solution/server/endpoint/routes/actions/isolation.test.ts

@@ -42,7 +42,7 @@ import {
   HostMetadata,
 } from '../../../../common/endpoint/types';
 import { EndpointDocGenerator } from '../../../../common/endpoint/generate_data';
-import { createV2SearchResponse } from '../metadata/support/test_support';
+import { legacyMetadataSearchResponse } from '../metadata/support/test_support';
 import { ElasticsearchAssetType } from '../../../../../fleet/common';
 import { CasesClientMock } from '../../../../../cases/server/client/mocks';
 
@@ -188,7 +188,7 @@ describe('Host Isolation', () => {
         ctx.core.elasticsearch.client.asCurrentUser.search = jest
           .fn()
           .mockImplementation(() =>
-            Promise.resolve({ body: createV2SearchResponse(searchResponse) })
+            Promise.resolve({ body: legacyMetadataSearchResponse(searchResponse) })
           );
         const withLicense = license ? license : Platinum;
         licenseEmitter.next(withLicense);