[Fleet] Use fleet server indices for enrollment keys and to list agents with a feature flag

x-pack/plugins/fleet/common/constants/agent.ts

@@ -22,3 +22,5 @@ export const AGENT_UPDATE_ACTIONS_INTERVAL_MS = 5000;
 
 export const AGENT_POLICY_ROLLOUT_RATE_LIMIT_INTERVAL_MS = 1000;
 export const AGENT_POLICY_ROLLOUT_RATE_LIMIT_REQUEST_PER_INTERVAL = 5;
+
+export const AGENTS_INDEX = '.fleet-agents';

x-pack/plugins/fleet/common/constants/agent_policy.ts

@@ -6,7 +6,7 @@
 import { defaultPackages } from './epm';
 import { AgentPolicy } from '../types';
 export const AGENT_POLICY_SAVED_OBJECT_TYPE = 'ingest-agent-policies';
-
+export const AGENT_POLICY_INDEX = '.fleet-policies';
 export const agentPolicyStatuses = {
   Active: 'active',
   Inactive: 'inactive',

x-pack/plugins/fleet/common/constants/enrollment_api_key.ts

@@ -5,3 +5,5 @@
  */
 
 export const ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE = 'fleet-enrollment-api-keys';
+
+export const ENROLLMENT_API_KEYS_INDEX = '.fleet-enrollment-api-keys';

x-pack/plugins/fleet/common/services/agent_status.ts

@@ -41,7 +41,7 @@ export function getAgentStatus(agent: Agent, now: number = Date.now()): AgentSta
 }
 
 export function buildKueryForEnrollingAgents() {
-  return `not ${AGENT_SAVED_OBJECT_TYPE}.last_checkin:*`;
+  return `not (${AGENT_SAVED_OBJECT_TYPE}.last_checkin:*)`;
 }
 
 export function buildKueryForUnenrollingAgents() {
@@ -53,7 +53,7 @@ export function buildKueryForOnlineAgents() {
 }
 
 export function buildKueryForErrorAgents() {
-  return `( ${AGENT_SAVED_OBJECT_TYPE}.last_checkin_status:error or ${AGENT_SAVED_OBJECT_TYPE}.last_checkin_status:degraded )`;
+  return `${AGENT_SAVED_OBJECT_TYPE}.last_checkin_status:error or ${AGENT_SAVED_OBJECT_TYPE}.last_checkin_status:degraded`;
 }
 
 export function buildKueryForOfflineAgents() {

x-pack/plugins/fleet/common/types/index.ts

@@ -11,6 +11,7 @@ export interface FleetConfigType {
   registryUrl?: string;
   registryProxyUrl?: string;
   agents: {
+    fleetServerEnabled: boolean;
     enabled: boolean;
     tlsCheckDisabled: boolean;
     pollingRequestTimeout: number;

x-pack/plugins/fleet/common/types/models/agent_policy.ts

@@ -80,3 +80,37 @@ export interface FullAgentPolicyKibanaConfig {
   protocol: string;
   path?: string;
 }
+
+// Generated from Fleet Server schema.json
+
+/**
+ * A policy that an Elastic Agent is attached to
+ */
+export interface FleetServerPolicy {
+  /**
+   * Date/time the policy revision was created
+   */
+  '@timestamp'?: string;
+  /**
+   * The ID of the policy
+   */
+  policy_id: string;
+  /**
+   * The revision index of the policy
+   */
+  revision_idx: number;
+  /**
+   * The coordinator index of the policy
+   */
+  coordinator_idx: number;
+  /**
+   * The opaque payload.
+   */
+  data: {
+    [k: string]: unknown;
+  };
+  /**
+   * True when this policy is the default policy to start Fleet Server
+   */
+  default_fleet_server: boolean;
+}

x-pack/plugins/fleet/common/types/models/enrollment_api_key.ts

@@ -15,3 +15,31 @@ export interface EnrollmentAPIKey {
 }
 
 export type EnrollmentAPIKeySOAttributes = Omit<EnrollmentAPIKey, 'id'>;
+
+// Generated
+
+/**
+ * An Elastic Agent enrollment API key
+ */
+export interface FleetServerEnrollmentAPIKey {
+  /**
+   * True when the key is active
+   */
+  active?: boolean;
+  /**
+   * The unique identifier for the enrollment key, currently xid
+   */
+  api_key_id: string;
+  /**
+   * Api key
+   */
+  api_key: string;
+  /**
+   * Enrollment key name
+   */
+  name?: string;
+  policy_id?: string;
+  expire_at?: string;
+  created_at?: string;
+  updated_at?: string;
+}

x-pack/plugins/fleet/public/applications/fleet/mock/plugin_configuration.ts

@@ -13,6 +13,7 @@ export const createConfigurationMock = (): FleetConfigType => {
     registryProxyUrl: '',
     agents: {
       enabled: true,
+      fleetServerEnabled: false,
       tlsCheckDisabled: true,
       pollingRequestTimeout: 1000,
       maxConcurrentConnections: 100,

x-pack/plugins/fleet/server/collectors/agent_collectors.ts

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { SavedObjectsClient } from 'kibana/server';
+import { ElasticsearchClient, SavedObjectsClient } from 'kibana/server';
 import * as AgentService from '../services/agents';
 export interface AgentUsage {
   total: number;
@@ -13,9 +13,12 @@ export interface AgentUsage {
   offline: number;
 }
 
-export const getAgentUsage = async (soClient?: SavedObjectsClient): Promise<AgentUsage> => {
+export const getAgentUsage = async (
+  soClient?: SavedObjectsClient,
+  esClient?: ElasticsearchClient
+): Promise<AgentUsage> => {
   // TODO: unsure if this case is possible at all.
-  if (!soClient) {
+  if (!soClient || !esClient) {
     return {
       total: 0,
       online: 0,
@@ -24,7 +27,8 @@ export const getAgentUsage = async (soClient?: SavedObjectsClient): Promise<Agen
     };
   }
   const { total, online, error, offline } = await AgentService.getAgentStatusForAgentPolicy(
-    soClient
+    soClient,
+    esClient
   );
   return {
     total,

x-pack/plugins/fleet/server/collectors/helpers.ts

@@ -5,11 +5,14 @@
  */
 
 import { CoreSetup } from 'kibana/server';
-import { SavedObjectsClient } from '../../../../../src/core/server';
+import { ElasticsearchClient, SavedObjectsClient } from '../../../../../src/core/server';
 
-export async function getInternalSavedObjectsClient(core: CoreSetup) {
+export async function getInternalClients(
+  core: CoreSetup
+): Promise<[SavedObjectsClient, ElasticsearchClient]> {
   return core.getStartServices().then(async ([coreStart]) => {
     const savedObjectsRepo = coreStart.savedObjects.createInternalRepository();
-    return new SavedObjectsClient(savedObjectsRepo);
+    const esClient = coreStart.elasticsearch.client.asInternalUser;
+    return [new SavedObjectsClient(savedObjectsRepo), esClient];
   });
 }

x-pack/plugins/fleet/server/collectors/register.ts

@@ -8,7 +8,7 @@ import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
 import { CoreSetup } from 'kibana/server';
 import { getIsAgentsEnabled } from './config_collectors';
 import { AgentUsage, getAgentUsage } from './agent_collectors';
-import { getInternalSavedObjectsClient } from './helpers';
+import { getInternalClients } from './helpers';
 import { PackageUsage, getPackageUsage } from './package_collectors';
 import { FleetConfigType } from '..';
 
@@ -34,10 +34,10 @@ export function registerFleetUsageCollector(
     type: 'fleet',
     isReady: () => true,
     fetch: async () => {
-      const soClient = await getInternalSavedObjectsClient(core);
+      const [soClient, esClient] = await getInternalClients(core);
       return {
         agents_enabled: getIsAgentsEnabled(config),
-        agents: await getAgentUsage(soClient),
+        agents: await getAgentUsage(soClient, esClient),
         packages: await getPackageUsage(soClient),
       };
     },

x-pack/plugins/fleet/server/constants/index.ts

@@ -47,4 +47,7 @@ export {
   // Defaults
   DEFAULT_AGENT_POLICY,
   DEFAULT_OUTPUT,
+  // Fleet Server index
+  ENROLLMENT_API_KEYS_INDEX,
+  AGENTS_INDEX,
 } from '../../common';

x-pack/plugins/fleet/server/index.ts

@@ -37,6 +37,7 @@ export const config: PluginConfigDescriptor = {
     registryProxyUrl: schema.maybe(schema.uri({ scheme: ['http', 'https'] })),
     agents: schema.object({
       enabled: schema.boolean({ defaultValue: true }),
+      fleetServerEnabled: schema.boolean({ defaultValue: false }),
       tlsCheckDisabled: schema.boolean({ defaultValue: false }),
       pollingRequestTimeout: schema.number({
         defaultValue: AGENT_POLLING_REQUEST_TIMEOUT_MS,

x-pack/plugins/fleet/server/mocks.ts

@@ -4,7 +4,11 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { loggingSystemMock, savedObjectsServiceMock } from 'src/core/server/mocks';
+import {
+  elasticsearchServiceMock,
+  loggingSystemMock,
+  savedObjectsServiceMock,
+} from 'src/core/server/mocks';
 import { FleetAppContext } from './plugin';
 import { encryptedSavedObjectsMock } from '../../encrypted_saved_objects/server/mocks';
 import { securityMock } from '../../security/server/mocks';
@@ -13,6 +17,7 @@ import { AgentPolicyServiceInterface, AgentService } from './services';
 
 export const createAppContextStartContractMock = (): FleetAppContext => {
   return {
+    elasticsearch: elasticsearchServiceMock.createStart(),
     encryptedSavedObjectsStart: encryptedSavedObjectsMock.createStart(),
     savedObjects: savedObjectsServiceMock.createStartContract(),
     security: securityMock.createStart(),

x-pack/plugins/fleet/server/plugin.ts

@@ -8,6 +8,7 @@ import { first } from 'rxjs/operators';
 import {
   CoreSetup,
   CoreStart,
+  ElasticsearchServiceStart,
   Logger,
   Plugin,
   PluginInitializerContext,
@@ -80,6 +81,7 @@ import { agentCheckinState } from './services/agents/checkin/state';
 import { registerFleetUsageCollector } from './collectors/register';
 import { getInstallation } from './services/epm/packages';
 import { makeRouterEnforcingSuperuser } from './routes/security';
+import { runFleetServerMigration } from './services/fleet_server_migration';
 
 export interface FleetSetupDeps {
   licensing: LicensingPluginSetup;
@@ -96,6 +98,7 @@ export interface FleetStartDeps {
 }
 
 export interface FleetAppContext {
+  elasticsearch: ElasticsearchServiceStart;
   encryptedSavedObjectsStart?: EncryptedSavedObjectsPluginStart;
   encryptedSavedObjectsSetup?: EncryptedSavedObjectsPluginSetup;
   security?: SecurityPluginStart;
@@ -276,6 +279,7 @@ export class FleetPlugin
 
   public async start(core: CoreStart, plugins: FleetStartDeps): Promise<FleetStartContract> {
     await appContextService.start({
+      elasticsearch: core.elasticsearch,
       encryptedSavedObjectsStart: plugins.encryptedSavedObjects,
       encryptedSavedObjectsSetup: this.encryptedSavedObjectsSetup,
       security: plugins.security,
@@ -291,6 +295,13 @@ export class FleetPlugin
     licenseService.start(this.licensing$);
     agentCheckinState.start();
 
+    const fleetServerEnabled = appContextService.getConfig()?.agents?.fleetServerEnabled;
+    if (fleetServerEnabled) {
+      // We need licence to be initialized before using the SO service.
+      await this.licensing$.pipe(first()).toPromise();
+      await runFleetServerMigration();
+    }
+
     return {
       esIndexPatternService: new ESIndexPatternSavedObjectService(),
       packageService: {

x-pack/plugins/fleet/server/routes/agent/acks_handlers.test.ts

@@ -6,11 +6,16 @@
 
 import { postAgentAcksHandlerBuilder } from './acks_handlers';
 import {
+  ElasticsearchClient,
   KibanaResponseFactory,
   RequestHandlerContext,
   SavedObjectsClientContract,
 } from 'kibana/server';
-import { httpServerMock, savedObjectsClientMock } from '../../../../../../src/core/server/mocks';
+import {
+  elasticsearchServiceMock,
+  httpServerMock,
+  savedObjectsClientMock,
+} from '../../../../../../src/core/server/mocks';
 import { PostAgentAcksResponse } from '../../../common/types/rest_spec';
 import { AckEventSchema } from '../../types/models';
 import { AcksService } from '../../services/agents';
@@ -45,9 +50,11 @@ describe('test acks schema', () => {
 describe('test acks handlers', () => {
   let mockResponse: jest.Mocked<KibanaResponseFactory>;
   let mockSavedObjectsClient: jest.Mocked<SavedObjectsClientContract>;
+  let mockElasticsearchClient: jest.Mocked<ElasticsearchClient>;
 
   beforeEach(() => {
     mockSavedObjectsClient = savedObjectsClientMock.create();
+    mockElasticsearchClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
     mockResponse = httpServerMock.createResponseFactory();
   });
 
@@ -81,6 +88,7 @@ describe('test acks handlers', () => {
         id: 'agent',
       }),
       getSavedObjectsClientContract: jest.fn().mockReturnValueOnce(mockSavedObjectsClient),
+      getElasticsearchClientContract: jest.fn().mockReturnValueOnce(mockElasticsearchClient),
       saveAgentEvents: jest.fn(),
     } as jest.Mocked<AcksService>;
 

x-pack/plugins/fleet/server/routes/agent/acks_handlers.ts

@@ -18,6 +18,7 @@ export const postAgentAcksHandlerBuilder = function (
   return async (context, request, response) => {
     try {
       const soClient = ackService.getSavedObjectsClientContract(request);
+      const esClient = ackService.getElasticsearchClientContract();
       const agent = await ackService.authenticateAgentWithAccessToken(soClient, request);
       const agentEvents = request.body.events as AgentEvent[];
 
@@ -33,7 +34,12 @@ export const postAgentAcksHandlerBuilder = function (
         });
       }
 
-      const agentActions = await ackService.acknowledgeAgentActions(soClient, agent, agentEvents);
+      const agentActions = await ackService.acknowledgeAgentActions(
+        soClient,
+        esClient,
+        agent,
+        agentEvents
+      );
 
       if (agentActions.length > 0) {
         await ackService.saveAgentEvents(soClient, agentEvents);

x-pack/plugins/fleet/server/routes/agent/actions_handlers.test.ts

@@ -6,11 +6,16 @@
 
 import { NewAgentActionSchema } from '../../types/models';
 import {
+  ElasticsearchClient,
   KibanaResponseFactory,
   RequestHandlerContext,
   SavedObjectsClientContract,
 } from 'kibana/server';
-import { savedObjectsClientMock, httpServerMock } from 'src/core/server/mocks';
+import {
+  elasticsearchServiceMock,
+  savedObjectsClientMock,
+  httpServerMock,
+} from 'src/core/server/mocks';
 import { ActionsService } from '../../services/agents';
 import { AgentAction } from '../../../common/types/models';
 import { postNewAgentActionHandlerBuilder } from './actions_handlers';
@@ -41,9 +46,11 @@ describe('test actions handlers schema', () => {
 describe('test actions handlers', () => {
   let mockResponse: jest.Mocked<KibanaResponseFactory>;
   let mockSavedObjectsClient: jest.Mocked<SavedObjectsClientContract>;
+  let mockElasticsearchClient: jest.Mocked<ElasticsearchClient>;
 
   beforeEach(() => {
     mockSavedObjectsClient = savedObjectsClientMock.create();
+    mockElasticsearchClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
     mockResponse = httpServerMock.createResponseFactory();
   });
 
@@ -84,6 +91,11 @@ describe('test actions handlers', () => {
           savedObjects: {
             client: mockSavedObjectsClient,
           },
+          elasticsearch: {
+            client: {
+              asInternalUser: mockElasticsearchClient,
+            },
+          },
         },
       } as unknown) as RequestHandlerContext,
       mockRequest,

x-pack/plugins/fleet/server/routes/agent/actions_handlers.ts

@@ -23,8 +23,9 @@ export const postNewAgentActionHandlerBuilder = function (
   return async (context, request, response) => {
     try {
       const soClient = context.core.savedObjects.client;
+      const esClient = context.core.elasticsearch.client.asInternalUser;
 
-      const agent = await actionsService.getAgent(soClient, request.params.agentId);
+      const agent = await actionsService.getAgent(soClient, esClient, request.params.agentId);
 
       const newAgentAction = request.body.action;