[UII] Remove references to .fleet-servers index (#183868)

## Summary Resolves #173537. This PR removes all usages and references to `.fleet-servers` index. A small refactoring was done that moves helper functions in enrollment settings API to fleet server services. It's a better place for them because they are related to fleet server anyway. With this change, we now use these functions to calculate the readiness of Fleet (`hasFleetServers` was previously reading on `.fleet-servers` index). The readiness of a fleet server is defined as at least one online agent enrolled into an agent policy that contains a fleet server policy. ### Changes in Security Solution (@paul-tavares) - Updated `enableFleetServerIfNecessary()` utility _(used for testing, dev)_ to not use `.fleet-servers` index and instead write a Agent record to the `.fleet-agents` index for Fleet Server. This record writing is skipped when running tests in serverless mode, instead, `xpack.fleet.internal.fleetServerStandalone=true` is added to mimic skipping checks for Fleet Server in real serverless. ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: Paul Tavares <[email protected]>
elastic · Jun 5, 2024 · dceae3e · dceae3e
1 parent 1772203
commit dceae3e
Show file tree

Hide file tree

Showing 30 changed files with 539 additions and 462 deletions.
diff --git a/x-pack/plugins/fleet/common/constants/index.ts b/x-pack/plugins/fleet/common/constants/index.ts
@@ -37,8 +37,6 @@ export const FLEET_SERVER_INDICES_VERSION = 1;
 
 export const FLEET_SERVER_ARTIFACTS_INDEX = '.fleet-artifacts';
 
-export const FLEET_SERVER_SERVERS_INDEX = '.fleet-servers';
-
 export const FLEET_SERVER_INDICES = [
   '.fleet-actions',
   '.fleet-actions-results',
@@ -47,7 +45,6 @@ export const FLEET_SERVER_INDICES = [
   '.fleet-enrollment-api-keys',
   '.fleet-policies',
   '.fleet-policies-leader',
-  FLEET_SERVER_SERVERS_INDEX,
 ];
 
 // Nodes that can be queried by datastreams API

diff --git a/x-pack/plugins/fleet/common/index.ts b/x-pack/plugins/fleet/common/index.ts
@@ -32,7 +32,6 @@ export {
   MESSAGE_SIGNING_KEYS_SAVED_OBJECT_TYPE,
   UNINSTALL_TOKENS_SAVED_OBJECT_TYPE,
   // Fleet server index
-  FLEET_SERVER_SERVERS_INDEX,
   FLEET_SERVER_ARTIFACTS_INDEX,
   AGENTS_INDEX,
   AGENT_POLICY_INDEX,

diff --git a/x-pack/plugins/fleet/cypress/e2e/agents/agent_list.cy.ts b/x-pack/plugins/fleet/cypress/e2e/agents/agent_list.cy.ts
@@ -9,7 +9,7 @@ import { FLEET_AGENT_LIST_PAGE } from '../../screens/fleet';
 
 import { createAgentDoc } from '../../tasks/agents';
 import { setupFleetServer } from '../../tasks/fleet_server';
-import { deleteFleetServerDocs, deleteAgentDocs, cleanupAgentPolicies } from '../../tasks/cleanup';
+import { deleteAgentDocs, cleanupAgentPolicies } from '../../tasks/cleanup';
 import type { CreateAgentPolicyRequest } from '../../../common/types';
 import { setUISettings } from '../../tasks/ui_settings';
 
@@ -87,7 +87,6 @@ function assertTableIsEmpty() {
 
 describe('View agents list', () => {
   before(() => {
-    deleteFleetServerDocs(true);
     deleteAgentDocs(true);
     cleanupAgentPolicies();
     setupFleetServer();
@@ -103,7 +102,6 @@ describe('View agents list', () => {
     }
   });
   after(() => {
-    deleteFleetServerDocs(true);
     deleteAgentDocs(true);
     cleanupAgentPolicies();
   });

diff --git a/x-pack/plugins/fleet/cypress/e2e/fleet_agent_flyout.cy.ts b/x-pack/plugins/fleet/cypress/e2e/fleet_agent_flyout.cy.ts
@@ -6,7 +6,7 @@
  */
 
 import { ADD_AGENT_BUTTON, AGENT_FLYOUT } from '../screens/fleet';
-import { cleanupAgentPolicies, deleteFleetServerDocs, deleteAgentDocs } from '../tasks/cleanup';
+import { cleanupAgentPolicies, deleteAgentDocs } from '../tasks/cleanup';
 import { createAgentDoc } from '../tasks/agents';
 import { setFleetServerHost } from '../tasks/fleet_server';
 import { FLEET, navigateTo } from '../tasks/navigation';
@@ -18,7 +18,6 @@ import { login } from '../tasks/login';
 const FLEET_SERVER_POLICY_ID = 'fleet-server-policy';
 
 function cleanUp() {
-  deleteFleetServerDocs(true);
   deleteAgentDocs(true);
   cleanupAgentPolicies();
 }
@@ -53,14 +52,6 @@ describe('Fleet add agent flyout', () => {
           index: '.fleet-agents',
           docs: [createAgentDoc('agent1', policyId, 'online', kibanaVersion)],
         });
-        cy.task('insertDocs', {
-          index: '.fleet-servers',
-          docs: [
-            {
-              '@timestamp': new Date().toISOString(),
-            },
-          ],
-        });
         setFleetServerHost();
       });
 

diff --git a/x-pack/plugins/fleet/cypress/e2e/fleet_startup.cy.ts b/x-pack/plugins/fleet/cypress/e2e/fleet_startup.cy.ts
@@ -19,7 +19,7 @@ import {
 import { cleanupAgentPolicies, unenrollAgent } from '../tasks/cleanup';
 import { request } from '../tasks/common';
 import { verifyPolicy, verifyAgentPackage, navigateToTab } from '../tasks/fleet';
-import { deleteFleetServer, setFleetServerHost } from '../tasks/fleet_server';
+import { setFleetServerHost } from '../tasks/fleet_server';
 import { login } from '../tasks/login';
 import { FLEET, navigateTo } from '../tasks/navigation';
 
@@ -28,8 +28,6 @@ describe('Fleet startup', () => {
     before(() => {
       unenrollAgent();
       cleanupAgentPolicies();
-      deleteFleetServer();
-
       setFleetServerHost();
     });
 

diff --git a/x-pack/plugins/fleet/cypress/tasks/cleanup.ts b/x-pack/plugins/fleet/cypress/tasks/cleanup.ts
@@ -48,13 +48,6 @@ export function cleanupDownloadSources() {
   });
 }
 
-export function deleteFleetServerDocs(ignoreUnavailable: boolean = false) {
-  cy.task('deleteDocsByQuery', {
-    index: '.fleet-servers',
-    query: { match_all: {} },
-    ignoreUnavailable,
-  });
-}
 export function deleteAgentDocs(ignoreUnavailable: boolean = false) {
   cy.task('deleteDocsByQuery', {
     index: '.fleet-agents',

diff --git a/x-pack/plugins/fleet/cypress/tasks/fleet_server.ts b/x-pack/plugins/fleet/cypress/tasks/fleet_server.ts
@@ -44,26 +44,10 @@ export async function setupFleetServer() {
       index: '.fleet-agents',
       docs: [createAgentDoc('fleet-server', policyId, 'online', kibanaVersion)],
     });
-    cy.task('insertDocs', {
-      index: '.fleet-servers',
-      docs: [
-        {
-          '@timestamp': new Date().toISOString(),
-        },
-      ],
-    });
     setFleetServerHost();
   });
 }
 
-export function deleteFleetServer() {
-  cy.task('deleteDocsByQuery', {
-    index: '.fleet-servers',
-    query: { match_all: {} },
-    ignoreUnavailable: true,
-  });
-}
-
 export function setFleetServerHost(host = 'https://fleetserver:8220') {
   request({
     method: 'POST',

diff --git a/x-pack/plugins/fleet/dev_docs/data_model.md b/x-pack/plugins/fleet/dev_docs/data_model.md
@@ -58,10 +58,6 @@ The total schema for actions is represented by the `FleetServerAgentAction` type
 
 - Cleanup model: N/A
 
-### `.fleet-servers`
-
-- Cleanup model: N/A
-
 ### `.fleet-artifacts`
 
 - Cleanup model: N/A

diff --git a/...lugins/fleet/public/applications/fleet/sections/debug/components/fleet_index_debugger.tsx b/...lugins/fleet/public/applications/fleet/sections/debug/components/fleet_index_debugger.tsx
@@ -43,7 +43,6 @@ export const FleetIndexDebugger = () => {
   const indices = [
     { label: '.fleet-agents', value: '.fleet-agents' },
     { label: '.fleet-actions', value: '.fleet-actions' },
-    { label: '.fleet-servers', value: '.fleet-servers' },
     { label: '.fleet-enrollment-api-keys', value: '.fleet-enrollment-api-keys' },
   ];
   const [index, setIndex] = useState<string | undefined>();

diff --git a/x-pack/plugins/fleet/server/constants/index.ts b/x-pack/plugins/fleet/server/constants/index.ts
@@ -58,7 +58,6 @@ export {
   PACKAGE_POLICY_DEFAULT_INDEX_PRIVILEGES,
   AGENT_POLICY_DEFAULT_MONITORING_DATASETS,
   // Fleet Server index
-  FLEET_SERVER_SERVERS_INDEX,
   ENROLLMENT_API_KEYS_INDEX,
   AGENTS_INDEX,
   // Preconfiguration

diff --git a/x-pack/plugins/fleet/server/routes/settings/enrollment_settings_handler.test.ts b/x-pack/plugins/fleet/server/routes/settings/enrollment_settings_handler.test.ts
@@ -4,21 +4,14 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks';
+import { savedObjectsClientMock } from '@kbn/core/server/mocks';
 
-import { packagePolicyService, agentPolicyService } from '../../services';
-import { getAgentStatusForAgentPolicy } from '../../services/agents';
+import { agentPolicyService } from '../../services';
+import { getFleetServerPolicies } from '../../services/fleet_server';
 
-import {
-  getFleetServerPolicies,
-  hasActiveFleetServersForPolicies,
-  getDownloadSource,
-} from './enrollment_settings_handler';
+import { getFleetServerOrAgentPolicies, getDownloadSource } from './enrollment_settings_handler';
 
 jest.mock('../../services', () => ({
-  packagePolicyService: {
-    list: jest.fn(),
-  },
   agentPolicyService: {
     get: jest.fn(),
     getByIDs: jest.fn(),
@@ -44,13 +37,12 @@ jest.mock('../../services', () => ({
   },
 }));
 
-jest.mock('../../services/agents', () => ({
-  getAgentStatusForAgentPolicy: jest.fn(),
+jest.mock('../../services/fleet_server', () => ({
+  getFleetServerPolicies: jest.fn(),
 }));
 
 describe('EnrollmentSettingsHandler utils', () => {
   const mockSoClient = savedObjectsClientMock.create();
-  const mockEsClient = elasticsearchServiceMock.createInternalClient();
   const mockAgentPolicies = [
     {
       id: 'agent-policy-1',
@@ -124,20 +116,21 @@ describe('EnrollmentSettingsHandler utils', () => {
     },
   ];
 
-  describe('getFleetServerPolicies', () => {
+  describe('getFleetServerOrAgentPolicies', () => {
     it('returns only fleet server policies if there are any when no agent policy ID is provided', async () => {
-      (packagePolicyService.list as jest.Mock).mockResolvedValueOnce({
-        items: [{ policy_id: 'fs-policy-1' }, { policy_id: 'fs-policy-2' }],
-      });
-      (agentPolicyService.getByIDs as jest.Mock).mockResolvedValueOnce(mockFleetServerPolicies);
-      const { fleetServerPolicies, scopedAgentPolicy } = await getFleetServerPolicies(mockSoClient);
+      (getFleetServerPolicies as jest.Mock).mockResolvedValueOnce(mockFleetServerPolicies);
+      const { fleetServerPolicies, scopedAgentPolicy } = await getFleetServerOrAgentPolicies(
+        mockSoClient
+      );
       expect(fleetServerPolicies).toEqual(mockFleetServerPolicies);
       expect(scopedAgentPolicy).toBeUndefined();
     });
 
     it('returns no fleet server policies when there are none and no agent policy ID is provided', async () => {
-      (packagePolicyService.list as jest.Mock).mockResolvedValueOnce({ items: [] });
-      const { fleetServerPolicies, scopedAgentPolicy } = await getFleetServerPolicies(mockSoClient);
+      (getFleetServerPolicies as jest.Mock).mockResolvedValueOnce([]);
+      const { fleetServerPolicies, scopedAgentPolicy } = await getFleetServerOrAgentPolicies(
+        mockSoClient
+      );
       expect(fleetServerPolicies).toEqual([]);
       expect(scopedAgentPolicy).toBeUndefined();
     });
@@ -147,7 +140,7 @@ describe('EnrollmentSettingsHandler utils', () => {
         ...mockFleetServerPolicies[1],
         package_policies: [mockPackagePolicies[1]],
       });
-      const { fleetServerPolicies, scopedAgentPolicy } = await getFleetServerPolicies(
+      const { fleetServerPolicies, scopedAgentPolicy } = await getFleetServerOrAgentPolicies(
         mockSoClient,
         'fs-policy-2'
       );
@@ -160,7 +153,7 @@ describe('EnrollmentSettingsHandler utils', () => {
         ...mockAgentPolicies[1],
         package_policies: [mockPackagePolicies[2]],
       });
-      const { fleetServerPolicies, scopedAgentPolicy } = await getFleetServerPolicies(
+      const { fleetServerPolicies, scopedAgentPolicy } = await getFleetServerOrAgentPolicies(
         mockSoClient,
         'agent-policy-2'
       );
@@ -170,7 +163,7 @@ describe('EnrollmentSettingsHandler utils', () => {
 
     it('returns no policies when specified agent policy ID is not found', async () => {
       (agentPolicyService.get as jest.Mock).mockResolvedValueOnce(undefined);
-      const { fleetServerPolicies, scopedAgentPolicy } = await getFleetServerPolicies(
+      const { fleetServerPolicies, scopedAgentPolicy } = await getFleetServerOrAgentPolicies(
         mockSoClient,
         'agent-policy-3'
       );
@@ -179,73 +172,6 @@ describe('EnrollmentSettingsHandler utils', () => {
     });
   });
 
-  describe('hasActiveFleetServersForPolicies', () => {
-    it('returns false when no agent IDs are provided', async () => {
-      const hasActive = await hasActiveFleetServersForPolicies(mockEsClient, mockSoClient, []);
-      expect(hasActive).toBe(false);
-    });
-
-    it('returns true when at least one agent is online', async () => {
-      (getAgentStatusForAgentPolicy as jest.Mock).mockResolvedValueOnce({
-        other: 0,
-        events: 0,
-        total: 1,
-        all: 1,
-        active: 0,
-        updating: 0,
-        offline: 0,
-        inactive: 0,
-        unenrolled: 0,
-        online: 1,
-        error: 0,
-      });
-      const hasActive = await hasActiveFleetServersForPolicies(mockEsClient, mockSoClient, [
-        'policy-1',
-      ]);
-      expect(hasActive).toBe(true);
-    });
-
-    it('returns true when at least one agent is updating', async () => {
-      (getAgentStatusForAgentPolicy as jest.Mock).mockResolvedValueOnce({
-        other: 0,
-        events: 0,
-        total: 1,
-        all: 1,
-        active: 0,
-        updating: 1,
-        offline: 0,
-        inactive: 0,
-        unenrolled: 0,
-        online: 0,
-        error: 0,
-      });
-      const hasActive = await hasActiveFleetServersForPolicies(mockEsClient, mockSoClient, [
-        'policy-1',
-      ]);
-      expect(hasActive).toBe(true);
-    });
-
-    it('returns false when no agents are updating or online', async () => {
-      (getAgentStatusForAgentPolicy as jest.Mock).mockResolvedValueOnce({
-        other: 0,
-        events: 0,
-        total: 3,
-        all: 3,
-        active: 1,
-        updating: 0,
-        offline: 1,
-        inactive: 1,
-        unenrolled: 1,
-        online: 0,
-        error: 1,
-      });
-      const hasActive = await hasActiveFleetServersForPolicies(mockEsClient, mockSoClient, [
-        'policy-1',
-      ]);
-      expect(hasActive).toBe(false);
-    });
-  });
-
   describe('getDownloadSource', () => {
     it('returns the default download source when no id is specified', async () => {
       const source = await getDownloadSource(mockSoClient);