From 5b62369318a1a1e5a2d7144538b2f9ae165473c7 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Wed, 24 Nov 2021 16:03:53 -0500 Subject: [PATCH] [7.17] Add missing package policies for managed preconfigured policies --- .../fleet/server/services/agent_policy.ts | 8 +- .../fleet/server/services/package_policy.ts | 19 ++- .../server/services/preconfiguration.test.ts | 140 ++++++++++++++++-- .../fleet/server/services/preconfiguration.ts | 115 +++++++------- 4 files changed, 204 insertions(+), 78 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts index 60ea9ee395096..692c53143c490 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.ts @@ -821,7 +821,8 @@ export async function addPackageToAgentPolicy( packagePolicyName?: string, packagePolicyId?: string | number, packagePolicyDescription?: string, - transformPackagePolicy?: (p: NewPackagePolicy) => NewPackagePolicy + transformPackagePolicy?: (p: NewPackagePolicy) => NewPackagePolicy, + bumpAgentPolicyRevison = false ) { const packageInfo = await getPackageInfo({ savedObjectsClient: soClient, @@ -850,7 +851,10 @@ export async function addPackageToAgentPolicy( await packagePolicyService.create(soClient, esClient, newPackagePolicy, { id, - bumpRevision: false, + bumpRevision: bumpAgentPolicyRevison, skipEnsureInstalled: true, + skipUniqueNameVerification: true, + overwrite: true, + force: true, // To add package to managed policy we need the force flag }); } diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 1ebb2faa0449c..c479b38cafae0 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -106,17 +106,22 @@ class PackagePolicyService { bumpRevision?: boolean; force?: boolean; skipEnsureInstalled?: boolean; + skipUniqueNameVerification?: boolean; + overwrite?: boolean; } ): Promise { - const existingPoliciesWithName = await this.list(soClient, { - perPage: 1, - kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.name: "${packagePolicy.name}"`, - }); + if (!options?.skipUniqueNameVerification) { + const existingPoliciesWithName = await this.list(soClient, { + perPage: 1, + kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.name: "${packagePolicy.name}"`, + }); - // Check that the name does not exist already - if (existingPoliciesWithName.items.length > 0) { - throw new IngestManagerError('There is already an integration policy with the same name'); + // Check that the name does not exist already + if (existingPoliciesWithName.items.length > 0) { + throw new IngestManagerError('There is already an integration policy with the same name'); + } } + let elasticsearch: PackagePolicy['elasticsearch']; // Add ids to stream const packagePolicyId = options?.id || uuid.v4(); diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts index aecbe079fac26..b64a28d00d661 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts @@ -5,12 +5,14 @@ * 2.0. */ +import uuid from 'uuid'; import { elasticsearchServiceMock, savedObjectsClientMock } from 'src/core/server/mocks'; import { SavedObjectsErrorHelpers } from '../../../../../src/core/server'; import type { InstallResult, + PackagePolicy, PreconfiguredAgentPolicy, PreconfiguredOutput, } from '../../common/types'; @@ -27,11 +29,13 @@ import { cleanPreconfiguredOutputs, } from './preconfiguration'; import { outputService } from './output'; +import { packagePolicyService } from './package_policy'; jest.mock('./agent_policy_update'); jest.mock('./output'); const mockedOutputService = outputService as jest.Mocked; +const mockedPackagePolicyService = packagePolicyService as jest.Mocked; const mockInstalledPackages = new Map(); const mockInstallPackageErrors = new Map(); @@ -56,7 +60,7 @@ function getPutPreconfiguredPackagesMock() { return { saved_objects: [ { - id: `mocked-${id}`, + id, attributes, type: type as string, score: 1, @@ -79,8 +83,9 @@ function getPutPreconfiguredPackagesMock() { soClient.get.mockImplementation(async (type, id) => { const attributes = mockConfiguredPolicies.get(id); if (!attributes) throw SavedObjectsErrorHelpers.createGenericNotFoundError(type, id); + return { - id: `mocked-${id}`, + id, attributes, type: type as string, references: [], @@ -91,7 +96,7 @@ function getPutPreconfiguredPackagesMock() { const { id } = options!; mockConfiguredPolicies.set(id, attributes); return { - id: `mocked-${id}`, + id: id || uuid.v4(), attributes, type, references: [], @@ -162,13 +167,15 @@ jest.mock('./package_policy', () => ({ packagePolicyService: { getByIDs: jest.fn().mockReturnValue([]), listIds: jest.fn().mockReturnValue({ items: [] }), - create(soClient: any, esClient: any, newPackagePolicy: NewPackagePolicy) { - return { - id: 'mocked', - version: 'mocked', - ...newPackagePolicy, - }; - }, + create: jest + .fn() + .mockImplementation((soClient: any, esClient: any, newPackagePolicy: NewPackagePolicy) => { + return { + id: 'mocked', + version: 'mocked', + ...newPackagePolicy, + }; + }), get(soClient: any, id: string) { return { id: 'mocked', @@ -200,6 +207,7 @@ const spyAgentPolicyServicBumpAllAgentPoliciesForOutput = jest.spyOn( describe('policy preconfiguration', () => { beforeEach(() => { + mockedPackagePolicyService.create.mockReset(); mockInstalledPackages.clear(); mockInstallPackageErrors.clear(); mockConfiguredPolicies.clear(); @@ -266,11 +274,116 @@ describe('policy preconfiguration', () => { ); expect(policies.length).toEqual(1); - expect(policies[0].id).toBe('mocked-test-id'); + expect(policies[0].id).toBe('test-id'); expect(packages).toEqual(expect.arrayContaining(['test_package-3.0.0'])); expect(nonFatalErrors.length).toBe(0); }); + it('should not add new package policy to existing non managed policies', async () => { + const soClient = getPutPreconfiguredPackagesMock(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + mockedPackagePolicyService.getByIDs.mockResolvedValue([ + { name: 'test_package1' } as PackagePolicy, + ]); + + mockConfiguredPolicies.set('test-id', { + name: 'Test policy', + description: 'Test policy description', + unenroll_timeout: 120, + namespace: 'default', + id: 'test-id', + package_policies: [ + { + name: 'test_package1', + }, + ], + } as PreconfiguredAgentPolicy); + + await ensurePreconfiguredPackagesAndPolicies( + soClient, + esClient, + [ + { + name: 'Test policy', + namespace: 'default', + id: 'test-id', + is_managed: false, + package_policies: [ + { + package: { name: 'test_package' }, + name: 'test_package1', + }, + { + package: { name: 'test_package' }, + name: 'test_package2', + }, + ], + }, + ] as PreconfiguredAgentPolicy[], + [{ name: 'test_package', version: '3.0.0' }], + mockDefaultOutput + ); + + expect(mockedPackagePolicyService.create).not.toBeCalled(); + }); + + it('should add new package policy to existing managed policies', async () => { + const soClient = getPutPreconfiguredPackagesMock(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + mockedPackagePolicyService.getByIDs.mockResolvedValue([ + { name: 'test_package1' } as PackagePolicy, + ]); + + mockConfiguredPolicies.set('test-id', { + name: 'Test policy', + description: 'Test policy description', + unenroll_timeout: 120, + namespace: 'default', + id: 'test-id', + package_policies: [ + { + name: 'test_package1', + }, + ], + is_managed: true, + } as PreconfiguredAgentPolicy); + + await ensurePreconfiguredPackagesAndPolicies( + soClient, + esClient, + [ + { + name: 'Test policy', + namespace: 'default', + id: 'test-id', + is_managed: true, + package_policies: [ + { + package: { name: 'test_package' }, + name: 'test_package1', + }, + { + package: { name: 'test_package' }, + name: 'test_package2', + }, + ], + }, + ] as PreconfiguredAgentPolicy[], + [{ name: 'test_package', version: '3.0.0' }], + mockDefaultOutput + ); + + expect(mockedPackagePolicyService.create).toBeCalledTimes(1); + expect(mockedPackagePolicyService.create).toBeCalledWith( + expect.anything(), // so client + expect.anything(), // es client + expect.objectContaining({ + name: 'test_package2', + }), + expect.anything() // options + ); + }); + it('should throw an error when trying to install duplicate packages', async () => { const soClient = getPutPreconfiguredPackagesMock(); const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; @@ -352,6 +465,7 @@ describe('policy preconfiguration', () => { '[Test policy] could not be added. [test_package] is not installed, add [test_package] to [xpack.fleet.packages] or remove it from [Test package].' ); }); + it('should not attempt to recreate or modify an agent policy if its ID is unchanged', async () => { const soClient = getPutPreconfiguredPackagesMock(); const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; @@ -373,7 +487,7 @@ describe('policy preconfiguration', () => { ); expect(policiesA.length).toEqual(1); - expect(policiesA[0].id).toBe('mocked-test-id'); + expect(policiesA[0].id).toBe('test-id'); expect(nonFatalErrorsA.length).toBe(0); const { policies: policiesB, nonFatalErrors: nonFatalErrorsB } = @@ -398,7 +512,7 @@ describe('policy preconfiguration', () => { ); expect(policiesB.length).toEqual(1); - expect(policiesB[0].id).toBe('mocked-test-id'); + expect(policiesB[0].id).toBe('test-id'); expect(policiesB[0].updated_at).toEqual(policiesA[0].updated_at); expect(nonFatalErrorsB.length).toBe(0); }); diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts index cd67cc85ef3d7..73f3b3d230863 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.ts @@ -19,12 +19,9 @@ import type { PreconfiguredPackage, PreconfigurationError, PreconfiguredOutput, + PackagePolicy, } from '../../common'; -import { - AGENT_POLICY_SAVED_OBJECT_TYPE, - SO_SEARCH_LIMIT, - normalizeHostsForAgents, -} from '../../common'; +import { SO_SEARCH_LIMIT, normalizeHostsForAgents } from '../../common'; import { PRECONFIGURATION_DELETION_RECORD_SAVED_OBJECT_TYPE, PRECONFIGURATION_LATEST_KEYWORD, @@ -287,65 +284,69 @@ export async function ensurePreconfiguredPackagesAndPolicies( } fulfilledPolicies.push(policyResult.value); const { created, policy, shouldAddIsManagedFlag } = policyResult.value; - if (created) { - try { - const preconfiguredAgentPolicy = policies[i]; - const { package_policies: packagePolicies } = preconfiguredAgentPolicy; - - const installedPackagePolicies = await Promise.all( - packagePolicies.map(async ({ package: pkg, name, ...newPackagePolicy }) => { - const installedPackage = await getInstallation({ - savedObjectsClient: soClient, - pkgName: pkg.name, - }); - if (!installedPackage) { - const rejectedPackage = rejectedPackages.find((rp) => rp.package?.name === pkg.name); - - if (rejectedPackage) { - throw new Error( - i18n.translate('xpack.fleet.preconfiguration.packageRejectedError', { - defaultMessage: `[{agentPolicyName}] could not be added. [{pkgName}] could not be installed due to error: [{errorMessage}]`, - values: { - agentPolicyName: preconfiguredAgentPolicy.name, - pkgName: pkg.name, - errorMessage: rejectedPackage.error.toString(), - }, - }) - ); - } + if (created || policies[i].is_managed) { + const preconfiguredAgentPolicy = policies[i]; + const { package_policies: packagePolicies } = preconfiguredAgentPolicy; + const agentPolicyWithPackagePolicies = await agentPolicyService.get( + soClient, + policy!.id, + true + ); + const installedPackagePolicies = await Promise.all( + packagePolicies.map(async ({ package: pkg, name, ...newPackagePolicy }) => { + const installedPackage = await getInstallation({ + savedObjectsClient: soClient, + pkgName: pkg.name, + }); + if (!installedPackage) { + const rejectedPackage = rejectedPackages.find((rp) => rp.package?.name === pkg.name); + + if (rejectedPackage) { throw new Error( - i18n.translate('xpack.fleet.preconfiguration.packageMissingError', { - defaultMessage: - '[{agentPolicyName}] could not be added. [{pkgName}] is not installed, add [{pkgName}] to [{packagesConfigValue}] or remove it from [{packagePolicyName}].', + i18n.translate('xpack.fleet.preconfiguration.packageRejectedError', { + defaultMessage: `[{agentPolicyName}] could not be added. [{pkgName}] could not be installed due to error: [{errorMessage}]`, values: { agentPolicyName: preconfiguredAgentPolicy.name, - packagePolicyName: name, pkgName: pkg.name, - packagesConfigValue: 'xpack.fleet.packages', + errorMessage: rejectedPackage.error.toString(), }, }) ); } - return { name, installedPackage, ...newPackagePolicy }; - }) - ); - await addPreconfiguredPolicyPackages( - soClient, - esClient, - policy!, - installedPackagePolicies!, - defaultOutput + + throw new Error( + i18n.translate('xpack.fleet.preconfiguration.packageMissingError', { + defaultMessage: + '[{agentPolicyName}] could not be added. [{pkgName}] is not installed, add [{pkgName}] to [{packagesConfigValue}] or remove it from [{packagePolicyName}].', + values: { + agentPolicyName: preconfiguredAgentPolicy.name, + packagePolicyName: name, + pkgName: pkg.name, + packagesConfigValue: 'xpack.fleet.packages', + }, + }) + ); + } + return { name, installedPackage, ...newPackagePolicy }; + }) + ); + + const packagePoliciesToAdd = installedPackagePolicies.filter((installablePackagePolicy) => { + return !(agentPolicyWithPackagePolicies?.package_policies as PackagePolicy[]).some( + (packagePolicy) => packagePolicy.name === installablePackagePolicy.name ); - // If ann error happens while adding a package to the policy we will delete the policy so the setup can be retried later - } catch (err) { - await soClient - .delete(AGENT_POLICY_SAVED_OBJECT_TYPE, policy!.id) - // swallow error - .catch((deleteErr) => appContextService.getLogger().error(deleteErr)); - - throw err; - } + }); + + await addPreconfiguredPolicyPackages( + soClient, + esClient, + policy!, + packagePoliciesToAdd!, + defaultOutput, + !created + ); + // Add the is_managed flag after configuring package policies to avoid errors if (shouldAddIsManagedFlag) { await agentPolicyService.update(soClient, esClient, policy!.id, { is_managed: true }); @@ -412,7 +413,8 @@ async function addPreconfiguredPolicyPackages( inputs?: InputsOverride[]; } >, - defaultOutput: Output + defaultOutput: Output, + bumpAgentPolicyRevison = false ) { // Add packages synchronously to avoid overwriting for (const { installedPackage, id, name, description, inputs } of installedPackagePolicies) { @@ -431,7 +433,8 @@ async function addPreconfiguredPolicyPackages( name, id, description, - (policy) => preconfigurePackageInputs(policy, packageInfo, inputs) + (policy) => overridePackageInputs(policy, packageInfo, inputs), + bumpAgentPolicyRevison ); } }