[Fleet] Support Fleet server system indices

x-pack/plugins/fleet/server/routes/agent/handlers.ts

@@ -317,34 +317,24 @@ export const postBulkAgentsReassignHandler: RequestHandler<
   const soClient = context.core.savedObjects.client;
   const esClient = context.core.elasticsearch.client.asInternalUser;
   try {
-    // Reassign by array of IDs
-    if (Array.isArray(request.body.agents)) {
-      await AgentService.reassignAgents(
-        soClient,
-        esClient,
-        { agentIds: request.body.agents },
-        request.body.policy_id
-      );
-    } else {
-      await AgentService.reassignAgents(
-        soClient,
-        esClient,
-        { kuery: request.body.agents },
-        request.body.policy_id
-      );
-    }
+    const results = await AgentService.reassignAgents(
+      soClient,
+      esClient,
+      Array.isArray(request.body.agents)
+        ? { agentIds: request.body.agents }
+        : { kuery: request.body.agents },
+      request.body.policy_id
+    );
 
-    const body: PostBulkAgentReassignResponse = {};
-    // TODO fix
-    // const body: PostBulkAgentReassignResponse = result.saved_objects.reduce((acc, so) => {
-    //   return {
-    //     ...acc,
-    //     [so.id]: {
-    //       success: !so.error,
-    //       error: so.error || undefined,
-    //     },
-    //   };
-    // }, {});
+    const body: PostBulkAgentReassignResponse = results.items.reduce((acc, so) => {
+      return {
+        ...acc,
+        [so.id]: {
+          success: !so.error,
+          error: so.error || undefined,
+        },
+      };
+    }, {});
     return response.ok({ body });
   } catch (error) {
     return defaultIngestErrorHandler({ error, response });

x-pack/plugins/fleet/server/services/agents/actions.ts

@@ -24,6 +24,8 @@ import {
 import { appContextService } from '../app_context';
 import { nodeTypes } from '../../../../../../src/plugins/data/common';
 
+const ONE_MONTH_IN_MS = 2592000000;
+
 export async function createAgentAction(
   soClient: SavedObjectsClientContract,
   esClient: ElasticsearchClient,
@@ -78,6 +80,7 @@ async function createAction(
   ) {
     const body: FleetServerAgentAction = {
       '@timestamp': new Date().toISOString(),
+      expiration: new Date(Date.now() + ONE_MONTH_IN_MS).toISOString(),
       agents: [actionSO.attributes.agent_id],
       action_id: actionSO.id,
       data: newAgentAction.data,
@@ -144,6 +147,7 @@ async function bulkCreateActions(
         }
         const body: FleetServerAgentAction = {
           '@timestamp': new Date().toISOString(),
+          expiration: new Date(Date.now() + ONE_MONTH_IN_MS).toISOString(),
           agents: [actionSO.attributes.agent_id],
           action_id: actionSO.id,
           data: actionSO.attributes.data ? JSON.parse(actionSO.attributes.data) : undefined,

x-pack/plugins/fleet/server/services/agents/checkin/state_new_actions.ts

@@ -106,25 +106,32 @@ function createAgentPolicyActionSharedObservable(agentPolicyId: string) {
   );
 }
 
-async function getOrCreateAgentDefaultOutputAPIKey(
+async function getAgentDefaultOutputAPIKey(
   soClient: SavedObjectsClientContract,
   esClient: ElasticsearchClient,
   agent: Agent
-): Promise<string> {
+) {
   if (appContextService.getConfig()?.agents?.fleetServerEnabled) {
-    if (agent.default_api_key) {
-      return agent.default_api_key;
-    }
+    return agent.default_api_key;
   } else {
     const {
       attributes: { default_api_key: defaultApiKey },
     } = await appContextService
       .getEncryptedSavedObjects()
       .getDecryptedAsInternalUser<AgentSOAttributes>(AGENT_SAVED_OBJECT_TYPE, agent.id);
 
-    if (defaultApiKey) {
-      return defaultApiKey;
-    }
+    return defaultApiKey;
+  }
+}
+
+async function getOrCreateAgentDefaultOutputAPIKey(
+  soClient: SavedObjectsClientContract,
+  esClient: ElasticsearchClient,
+  agent: Agent
+): Promise<string> {
+  const defaultAPIKey = await getAgentDefaultOutputAPIKey(soClient, esClient, agent);
+  if (defaultAPIKey) {
+    return defaultAPIKey;
   }
 
   const outputAPIKey = await APIKeysService.generateOutputApiKey(soClient, 'default', agent.id);

x-pack/plugins/fleet/server/services/agents/crud.ts

@@ -4,6 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
+
 import { SavedObjectsClientContract, ElasticsearchClient } from 'src/core/server';
 import { AgentSOAttributes, Agent, ListWithKuery } from '../../types';
 import { appContextService } from '../../services';

x-pack/plugins/fleet/server/services/agents/crud_fleet_server.ts

@@ -6,6 +6,7 @@
  */
 
 import Boom from '@hapi/boom';
+import { SearchResponse } from 'elasticsearch';
 import { ElasticsearchClient } from 'src/core/server';
 
 import { FleetServerAgent, isAgentUpgradeable, SO_SEARCH_LIMIT } from '../../../common';
@@ -17,32 +18,6 @@ import { searchHitToAgent, agentSOAttributesToFleetServerAgentDoc } from './help
 import { appContextService } from '../../services';
 import { esKuery } from '../../../../../../src/plugins/data/server';
 
-interface SearchResponse<T> {
-  took: number;
-  timed_out: boolean;
-  _scroll_id?: string;
-  hits: {
-    total: {
-      value: number;
-      relation: string;
-    };
-    max_score: number;
-    hits: Array<{
-      _index: string;
-      _type: string;
-      _id: string;
-      _score: number;
-      _source: T;
-      _version?: number;
-      fields?: any;
-      highlight?: any;
-      inner_hits?: any;
-      matched_queries?: string[];
-      sort?: string[];
-    }>;
-  };
-}
-
 const ACTIVE_AGENT_CONDITION = 'active:true';
 const INACTIVE_AGENT_CONDITION = `NOT (${ACTIVE_AGENT_CONDITION})`;
 
@@ -235,15 +210,23 @@ export async function bulkUpdateAgents(
       },
     },
     {
-      doc: agentSOAttributesToFleetServerAgentDoc(data),
+      doc: { ...agentSOAttributesToFleetServerAgentDoc(data) },
     },
   ]);
 
-  await esClient.bulk({
+  const res = await esClient.bulk({
     body,
     index: AGENTS_INDEX,
     refresh: 'wait_for',
   });
+
+  return {
+    items: res.body.items.map((item: { update: { _id: string; error?: Error } }) => ({
+      id: item.update._id,
+      success: !item.update.error,
+      error: item.update.error,
+    })),
+  };
 }
 
 export async function deleteAgent(esClient: ElasticsearchClient, agentId: string) {

x-pack/plugins/fleet/server/services/agents/crud_so.ts

@@ -217,7 +217,15 @@ export async function bulkUpdateAgents(
     })
   );
 
-  await soClient.bulkUpdate<AgentSOAttributes>(updates);
+  const res = await soClient.bulkUpdate<AgentSOAttributes>(updates);
+
+  return {
+    items: res.saved_objects.map((so) => ({
+      id: so.id,
+      success: !so.error,
+      error: so.error,
+    })),
+  };
 }
 
 export async function deleteAgent(soClient: SavedObjectsClientContract, agentId: string) {

x-pack/plugins/fleet/server/services/agents/reassign.ts

@@ -45,7 +45,7 @@ export async function reassignAgents(
         kuery: string;
       },
   newAgentPolicyId: string
-) {
+): Promise<{ items: Array<{ id: string; sucess: boolean; error?: Error }> }> {
   const agentPolicy = await agentPolicyService.get(soClient, newAgentPolicyId);
   if (!agentPolicy) {
     throw Boom.notFound(`Agent policy not found: ${newAgentPolicyId}`);

x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key_fleet_server.ts

@@ -7,43 +7,16 @@
 
 import uuid from 'uuid';
 import Boom from '@hapi/boom';
+import { GetResponse } from 'elasticsearch';
 import { ResponseError } from '@elastic/elasticsearch/lib/errors';
 import { SavedObjectsClientContract, ElasticsearchClient } from 'src/core/server';
+import { ESSearchResponse as SearchResponse } from '../../../../../typings/elasticsearch';
 import { EnrollmentAPIKey, FleetServerEnrollmentAPIKey } from '../../types';
 import { ENROLLMENT_API_KEYS_INDEX } from '../../constants';
 import { createAPIKey, invalidateAPIKeys } from './security';
 import { agentPolicyService } from '../agent_policy';
 import { escapeSearchQueryPhrase } from '../saved_object';
 
-// TODO Move these types to another file
-interface SearchResponse<T> {
-  took: number;
-  timed_out: boolean;
-  _scroll_id?: string;
-  hits: {
-    total: {
-      value: number;
-      relation: string;
-    };
-    max_score: number;
-    hits: Array<{
-      _index: string;
-      _type: string;
-      _id: string;
-      _score: number;
-      _source: T;
-      _version?: number;
-      fields?: any;
-      highlight?: any;
-      inner_hits?: any;
-      matched_queries?: string[];
-      sort?: string[];
-    }>;
-  };
-}
-
-type SearchHit<T> = SearchResponse<T>['hits']['hits'][0];
-
 export async function listEnrollmentApiKeys(
   esClient: ElasticsearchClient,
   options: {
@@ -55,7 +28,7 @@ export async function listEnrollmentApiKeys(
 ): Promise<{ items: EnrollmentAPIKey[]; total: any; page: any; perPage: any }> {
   const { page = 1, perPage = 20, kuery } = options;
 
-  const res = await esClient.search<SearchResponse<FleetServerEnrollmentAPIKey>>({
+  const res = await esClient.search<SearchResponse<FleetServerEnrollmentAPIKey, {}>>({
     index: ENROLLMENT_API_KEYS_INDEX,
     from: (page - 1) * perPage,
     size: perPage,
@@ -79,7 +52,7 @@ export async function getEnrollmentAPIKey(
   id: string
 ): Promise<EnrollmentAPIKey> {
   try {
-    const res = await esClient.get<SearchHit<FleetServerEnrollmentAPIKey>>({
+    const res = await esClient.get<GetResponse<FleetServerEnrollmentAPIKey>>({
       index: ENROLLMENT_API_KEYS_INDEX,
       id,
     });
@@ -187,7 +160,7 @@ export async function generateEnrollmentAPIKey(
 }
 
 export async function getEnrollmentAPIKeyById(esClient: ElasticsearchClient, apiKeyId: string) {
-  const res = await esClient.search<SearchResponse<FleetServerEnrollmentAPIKey>>({
+  const res = await esClient.search<SearchResponse<FleetServerEnrollmentAPIKey, {}>>({
     index: ENROLLMENT_API_KEYS_INDEX,
     q: `api_key_id:${escapeSearchQueryPhrase(apiKeyId)}`,
   });
@@ -212,7 +185,10 @@ async function validateAgentPolicyId(soClient: SavedObjectsClientContract, agent
   }
 }
 
-function esDocToEnrollmentApiKey(doc: SearchHit<FleetServerEnrollmentAPIKey>): EnrollmentAPIKey {
+function esDocToEnrollmentApiKey(doc: {
+  _id: string;
+  _source: FleetServerEnrollmentAPIKey;
+}): EnrollmentAPIKey {
   return {
     id: doc._id,
     ...doc._source,