[Fleet] Setup fleet server indices in Kibana without packages (#90658) (

#91543)

x-pack/plugins/fleet/common/constants/index.ts

@@ -22,6 +22,8 @@ export * from './settings';
 // setting in the future?
 export const SO_SEARCH_LIMIT = 10000;
 
+export const FLEET_SERVER_INDICES_VERSION = 1;
+
 export const FLEET_SERVER_INDICES = [
   '.fleet-actions',
   '.fleet-agents',

x-pack/plugins/fleet/server/collectors/agent_collectors.ts

@@ -7,7 +7,8 @@
 
 import { ElasticsearchClient, SavedObjectsClient } from 'kibana/server';
 import * as AgentService from '../services/agents';
-import { isFleetServerSetup } from '../services/fleet_server_migration';
+import { isFleetServerSetup } from '../services/fleet_server';
+
 export interface AgentUsage {
   total: number;
   online: number;

x-pack/plugins/fleet/server/plugin.ts

@@ -83,7 +83,7 @@ import { agentCheckinState } from './services/agents/checkin/state';
 import { registerFleetUsageCollector } from './collectors/register';
 import { getInstallation } from './services/epm/packages';
 import { makeRouterEnforcingSuperuser } from './routes/security';
-import { isFleetServerSetup } from './services/fleet_server_migration';
+import { startFleetServerSetup } from './services/fleet_server';
 
 export interface FleetSetupDeps {
   licensing: LicensingPluginSetup;
@@ -297,18 +297,9 @@ export class FleetPlugin
     licenseService.start(this.licensing$);
     agentCheckinState.start();
 
-    const fleetServerEnabled = appContextService.getConfig()?.agents?.fleetServerEnabled;
-    if (fleetServerEnabled) {
-      // We need licence to be initialized before using the SO service.
-      await this.licensing$.pipe(first()).toPromise();
-
-      const fleetSetup = await isFleetServerSetup();
-
-      if (!fleetSetup) {
-        this.logger?.warn(
-          'Extra setup is needed to be able to use central management for agent, please visit the Fleet app in Kibana.'
-        );
-      }
+    if (appContextService.getConfig()?.agents?.fleetServerEnabled) {
+      // Break the promise chain, the error handling is done in startFleetServerSetup
+      startFleetServerSetup();
     }
 
     return {

x-pack/plugins/fleet/server/services/app_context.ts

@@ -80,6 +80,10 @@ class AppContextService {
     return this.security;
   }
 
+  public hasSecurity() {
+    return !!this.security;
+  }
+
   public getCloud() {
     return this.cloud;
   }

x-pack/plugins/fleet/server/services/fleet_server/elastic_index.test.ts

@@ -0,0 +1,156 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { elasticsearchServiceMock } from 'src/core/server/mocks';
+import hash from 'object-hash';
+import { setupFleetServerIndexes } from './elastic_index';
+import ESFleetAgentIndex from './elasticsearch/fleet_agents.json';
+import ESFleetPoliciesIndex from './elasticsearch/fleet_policies.json';
+import ESFleetPoliciesLeaderIndex from './elasticsearch/fleet_policies_leader.json';
+import ESFleetServersIndex from './elasticsearch/fleet_servers.json';
+import ESFleetEnrollmentApiKeysIndex from './elasticsearch/fleet_enrollment_api_keys.json';
+import EsFleetActionsIndex from './elasticsearch/fleet_actions.json';
+
+const FLEET_INDEXES_MIGRATION_HASH = {
+  '.fleet-actions': hash(EsFleetActionsIndex),
+  '.fleet-agents': hash(ESFleetAgentIndex),
+  '.fleet-enrollment-apy-keys': hash(ESFleetEnrollmentApiKeysIndex),
+  '.fleet-policies': hash(ESFleetPoliciesIndex),
+  '.fleet-policies-leader': hash(ESFleetPoliciesLeaderIndex),
+  '.fleet-servers': hash(ESFleetServersIndex),
+};
+
+describe('setupFleetServerIndexes ', () => {
+  it('should create all the indices and aliases if nothings exists', async () => {
+    const esMock = elasticsearchServiceMock.createInternalClient();
+    await setupFleetServerIndexes(esMock);
+
+    const indexesCreated = esMock.indices.create.mock.calls.map((call) => call[0].index).sort();
+    expect(indexesCreated).toEqual([
+      '.fleet-actions_1',
+      '.fleet-agents_1',
+      '.fleet-enrollment-api-keys_1',
+      '.fleet-policies-leader_1',
+      '.fleet-policies_1',
+      '.fleet-servers_1',
+    ]);
+    const aliasesCreated = esMock.indices.updateAliases.mock.calls
+      .map((call) => (call[0].body as any)?.actions[0].add.alias)
+      .sort();
+
+    expect(aliasesCreated).toEqual([
+      '.fleet-actions',
+      '.fleet-agents',
+      '.fleet-enrollment-api-keys',
+      '.fleet-policies',
+      '.fleet-policies-leader',
+      '.fleet-servers',
+    ]);
+  });
+
+  it('should not create any indices and create aliases if indices exists but not the aliases', async () => {
+    const esMock = elasticsearchServiceMock.createInternalClient();
+    // @ts-expect-error
+    esMock.indices.exists.mockResolvedValue({ body: true });
+    // @ts-expect-error
+    esMock.indices.getMapping.mockImplementation((params: { index: string }) => {
+      return {
+        body: {
+          [params.index]: {
+            mappings: {
+              _meta: {
+                // @ts-expect-error
+                migrationHash: FLEET_INDEXES_MIGRATION_HASH[params.index.replace(/_1$/, '')],
+              },
+            },
+          },
+        },
+      };
+    });
+
+    await setupFleetServerIndexes(esMock);
+
+    expect(esMock.indices.create).not.toBeCalled();
+    const aliasesCreated = esMock.indices.updateAliases.mock.calls
+      .map((call) => (call[0].body as any)?.actions[0].add.alias)
+      .sort();
+
+    expect(aliasesCreated).toEqual([
+      '.fleet-actions',
+      '.fleet-agents',
+      '.fleet-enrollment-api-keys',
+      '.fleet-policies',
+      '.fleet-policies-leader',
+      '.fleet-servers',
+    ]);
+  });
+
+  it('should put new indices mapping if the mapping has been updated ', async () => {
+    const esMock = elasticsearchServiceMock.createInternalClient();
+    // @ts-expect-error
+    esMock.indices.exists.mockResolvedValue({ body: true });
+    // @ts-expect-error
+    esMock.indices.getMapping.mockImplementation((params: { index: string }) => {
+      return {
+        body: {
+          [params.index]: {
+            mappings: {
+              _meta: {
+                migrationHash: 'NOT_VALID_HASH',
+              },
+            },
+          },
+        },
+      };
+    });
+
+    await setupFleetServerIndexes(esMock);
+
+    expect(esMock.indices.create).not.toBeCalled();
+    const indexesMappingUpdated = esMock.indices.putMapping.mock.calls
+      .map((call) => call[0].index)
+      .sort();
+
+    expect(indexesMappingUpdated).toEqual([
+      '.fleet-actions_1',
+      '.fleet-agents_1',
+      '.fleet-enrollment-api-keys_1',
+      '.fleet-policies-leader_1',
+      '.fleet-policies_1',
+      '.fleet-servers_1',
+    ]);
+  });
+
+  it('should not create any indices or aliases if indices and aliases already exists', async () => {
+    const esMock = elasticsearchServiceMock.createInternalClient();
+
+    // @ts-expect-error
+    esMock.indices.exists.mockResolvedValue({ body: true });
+    // @ts-expect-error
+    esMock.indices.getMapping.mockImplementation((params: { index: string }) => {
+      return {
+        body: {
+          [params.index]: {
+            mappings: {
+              _meta: {
+                // @ts-expect-error
+                migrationHash: FLEET_INDEXES_MIGRATION_HASH[params.index.replace(/_1$/, '')],
+              },
+            },
+          },
+        },
+      };
+    });
+    // @ts-expect-error
+    esMock.indices.existsAlias.mockResolvedValue({ body: true });
+
+    await setupFleetServerIndexes(esMock);
+
+    expect(esMock.indices.create).not.toBeCalled();
+    expect(esMock.indices.updateAliases).not.toBeCalled();
+  });
+});

x-pack/plugins/fleet/server/services/fleet_server/elastic_index.ts

@@ -0,0 +1,117 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient } from 'kibana/server';
+import hash from 'object-hash';
+
+import { FLEET_SERVER_INDICES, FLEET_SERVER_INDICES_VERSION } from '../../../common';
+import { appContextService } from '../app_context';
+import ESFleetAgentIndex from './elasticsearch/fleet_agents.json';
+import ESFleetPoliciesIndex from './elasticsearch/fleet_policies.json';
+import ESFleetPoliciesLeaderIndex from './elasticsearch/fleet_policies_leader.json';
+import ESFleetServersIndex from './elasticsearch/fleet_servers.json';
+import ESFleetEnrollmentApiKeysIndex from './elasticsearch/fleet_enrollment_api_keys.json';
+import EsFleetActionsIndex from './elasticsearch/fleet_actions.json';
+
+const FLEET_INDEXES: Array<[typeof FLEET_SERVER_INDICES[number], any]> = [
+  ['.fleet-actions', EsFleetActionsIndex],
+  ['.fleet-agents', ESFleetAgentIndex],
+  ['.fleet-enrollment-api-keys', ESFleetEnrollmentApiKeysIndex],
+  ['.fleet-policies', ESFleetPoliciesIndex],
+  ['.fleet-policies-leader', ESFleetPoliciesLeaderIndex],
+  ['.fleet-servers', ESFleetServersIndex],
+];
+
+export async function setupFleetServerIndexes(
+  esClient = appContextService.getInternalUserESClient()
+) {
+  await Promise.all(
+    FLEET_INDEXES.map(async ([indexAlias, indexData]) => {
+      const index = `${indexAlias}_${FLEET_SERVER_INDICES_VERSION}`;
+      await createOrUpdateIndex(esClient, index, indexData);
+      await createAliasIfDoNotExists(esClient, indexAlias, index);
+    })
+  );
+}
+
+export async function createAliasIfDoNotExists(
+  esClient: ElasticsearchClient,
+  alias: string,
+  index: string
+) {
+  const { body: exists } = await esClient.indices.existsAlias({
+    name: alias,
+  });
+
+  if (exists === true) {
+    return;
+  }
+  await esClient.indices.updateAliases({
+    body: {
+      actions: [
+        {
+          add: { index, alias },
+        },
+      ],
+    },
+  });
+}
+
+async function createOrUpdateIndex(
+  esClient: ElasticsearchClient,
+  indexName: string,
+  indexData: any
+) {
+  const resExists = await esClient.indices.exists({
+    index: indexName,
+  });
+
+  // Support non destructive migration only (adding new field)
+  if (resExists.body === true) {
+    return updateIndex(esClient, indexName, indexData);
+  }
+
+  return createIndex(esClient, indexName, indexData);
+}
+
+async function updateIndex(esClient: ElasticsearchClient, indexName: string, indexData: any) {
+  const res = await esClient.indices.getMapping({
+    index: indexName,
+  });
+
+  const migrationHash = hash(indexData);
+  if (res.body[indexName].mappings?._meta?.migrationHash !== migrationHash) {
+    await esClient.indices.putMapping({
+      index: indexName,
+      body: Object.assign({
+        ...indexData.mappings,
+        _meta: { ...(indexData.mappings._meta || {}), migrationHash },
+      }),
+    });
+  }
+}
+
+async function createIndex(esClient: ElasticsearchClient, indexName: string, indexData: any) {
+  try {
+    const migrationHash = hash(indexData);
+    await esClient.indices.create({
+      index: indexName,
+      body: {
+        ...indexData,
+        mappings: Object.assign({
+          ...indexData.mappings,
+          _meta: { ...(indexData.mappings._meta || {}), migrationHash },
+        }),
+      },
+    });
+  } catch (err) {
+    // Swallow already exists errors as concurent Kibana can try to create that indice
+    if (err?.body?.error?.type !== 'resource_already_exists_exception') {
+      throw err;
+    }
+  }
+}

x-pack/plugins/fleet/server/services/fleet_server/elasticsearch/fleet_actions.json

@@ -0,0 +1,30 @@
+{
+  "settings": {},
+  "mappings": {
+    "dynamic": false,
+    "properties": {
+      "action_id": {
+        "type": "keyword"
+      },
+      "agents": {
+        "type": "keyword"
+      },
+      "data": {
+        "enabled": false,
+        "type": "object"
+      },
+      "expiration": {
+        "type": "date"
+      },
+      "input_type": {
+        "type": "keyword"
+      },
+      "@timestamp": {
+        "type": "date"
+      },
+      "type": {
+        "type": "keyword"
+      }
+    }
+  }
+}