diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/unused_types.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/unused_types.ts index 46ddd23251217..2f9dd57afa294 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/unused_types.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/unused_types.ts @@ -47,6 +47,8 @@ export const REMOVED_TYPES: string[] = [ 'csp_rule', // Removed in 8.8 https://github.com/elastic/kibana/pull/151116 'upgrade-assistant-telemetry', + // Removed in 8.8 https://github.com/elastic/kibana/pull/155204 + 'endpoint:user-artifact', ].sort(); export const excludeUnusedTypesQuery: QueryDslQueryContainer = { diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts index b585f8fc06a91..e3a54b835f008 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts @@ -81,7 +81,6 @@ describe('checking migration metadata changes on all registered SO types', () => "core-usage-stats": "b3c04da317c957741ebcdedfea4524049fdc79ff", "csp-rule-template": "099c229bf97578d9ca72b3a672d397559b84ee0b", "dashboard": "71e3f8dfcffeb5fbd410dec81ce46f5691763c43", - "endpoint:user-artifact": "1bc8f4ab74f9bb0d90239a431d435c16f31cc32e", "endpoint:user-artifact-manifest": "17d17827606db55e7499b1dab259263ffee3bb09", "enterprise_search_telemetry": "4b41830e3b28a16eb92dee0736b44ae6276ced9b", "epm-packages": "83235af7c95fd9bfb1d70996a5511e05b3fcc9ef", diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrate_artifacts_to_fleet.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrate_artifacts_to_fleet.test.ts deleted file mode 100644 index 277772253b92e..0000000000000 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrate_artifacts_to_fleet.test.ts +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { - loggingSystemMock, - savedObjectsClientMock, - elasticsearchServiceMock, -} from '@kbn/core/server/mocks'; -import type { - SavedObjectsClient, - Logger, - SavedObjectsFindResponse, - SavedObjectsFindResult, -} from '@kbn/core/server'; -import { migrateArtifactsToFleet } from './migrate_artifacts_to_fleet'; -import { createEndpointArtifactClientMock } from '../../services/artifacts/mocks'; -import type { InternalArtifactCompleteSchema } from '../../schemas'; -import { generateArtifactEsGetSingleHitMock } from '@kbn/fleet-plugin/server/services/artifacts/mocks'; -import type { NewArtifact } from '@kbn/fleet-plugin/server/services'; -import type { CreateRequest } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; - -describe('When migrating artifacts to fleet', () => { - let soClient: jest.Mocked; - let logger: jest.Mocked; - let artifactClient: ReturnType; - /** An artifact that was created prior to 7.14 */ - let soArtifactEntry: InternalArtifactCompleteSchema; - - const createSoFindResult = ( - soHits: SavedObjectsFindResult[] = [], - total: number = 15, - page: number = 1 - ): SavedObjectsFindResponse => { - return { - total, - page, - per_page: 10, - saved_objects: soHits, - }; - }; - - beforeEach(async () => { - soClient = savedObjectsClientMock.create() as unknown as jest.Mocked; - logger = loggingSystemMock.create().get() as jest.Mocked; - artifactClient = createEndpointArtifactClientMock(); - // pre-v7.14 artifact, which is compressed - soArtifactEntry = { - identifier: 'endpoint-exceptionlist-macos-v1', - compressionAlgorithm: 'zlib', - encryptionAlgorithm: 'none', - decodedSha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - encodedSha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', - decodedSize: 14, - encodedSize: 22, - body: 'eJyrVkrNKynKTC1WsoqOrQUAJxkFKQ==', - }; - - // Mock the esClient create response to include the artifact properties that were provide - // to it by fleet artifact client - artifactClient._esClient.create.mockImplementation((props: CreateRequest) => { - return elasticsearchServiceMock.createSuccessTransportRequestPromise({ - ...generateArtifactEsGetSingleHitMock({ - ...((props?.body ?? {}) as NewArtifact), - }), - _index: '.fleet-artifacts-7', - _id: `endpoint:endpoint-exceptionlist-macos-v1-${ - // @ts-expect-error TS2339 - props?.body?.decodedSha256 ?? 'UNKNOWN?' - }`, - _version: 1, - result: 'created', - _shards: { - total: 1, - successful: 1, - failed: 0, - }, - _seq_no: 0, - _primary_term: 1, - }); - }); - - soClient.find.mockResolvedValue(createSoFindResult([], 0)).mockResolvedValueOnce( - createSoFindResult([ - { - score: 1, - type: '', - id: 'abc123', - references: [], - attributes: soArtifactEntry, - }, - ]) - ); - }); - - it('should do nothing if there are no artifacts', async () => { - soClient.find.mockReset(); - soClient.find.mockResolvedValue(createSoFindResult([], 0)); - await migrateArtifactsToFleet(soClient, artifactClient, logger); - expect(soClient.find).toHaveBeenCalled(); - expect(artifactClient.createArtifact).not.toHaveBeenCalled(); - expect(soClient.delete).not.toHaveBeenCalled(); - }); - - it('should create new artifact via fleet client and delete prior SO one', async () => { - await migrateArtifactsToFleet(soClient, artifactClient, logger); - expect(artifactClient.createArtifact).toHaveBeenCalled(); - expect(soClient.delete).toHaveBeenCalled(); - }); - - it('should create artifact in fleet with attributes that match the SO version', async () => { - await migrateArtifactsToFleet(soClient, artifactClient, logger); - - await expect(artifactClient.createArtifact.mock.results[0].value).resolves.toEqual( - expect.objectContaining({ - ...soArtifactEntry, - compressionAlgorithm: 'zlib', - }) - ); - }); - - it('should ignore 404 responses for SO delete (multi-node kibana setup)', async () => { - const notFoundError: Error & { output?: { statusCode: number } } = new Error('not found'); - notFoundError.output = { statusCode: 404 }; - soClient.delete.mockRejectedValue(notFoundError); - await expect(migrateArtifactsToFleet(soClient, artifactClient, logger)).resolves.toEqual( - undefined - ); - expect(logger.debug).toHaveBeenCalledWith( - 'Artifact Migration: Attempt to delete Artifact SO [abc123] returned 404' - ); - }); - - it('should Throw() and log error if migration fails', async () => { - const error = new Error('test: delete failed'); - soClient.delete.mockRejectedValue(error); - await expect(migrateArtifactsToFleet(soClient, artifactClient, logger)).rejects.toThrow( - 'Artifact SO migration failed' - ); - }); -}); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrate_artifacts_to_fleet.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrate_artifacts_to_fleet.ts deleted file mode 100644 index e015019fa8a5d..0000000000000 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrate_artifacts_to_fleet.ts +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { inflate as _inflate } from 'zlib'; -import { promisify } from 'util'; -import type { SavedObjectsClient, Logger } from '@kbn/core/server'; -import type { EndpointArtifactClientInterface } from '../../services'; -import type { InternalArtifactCompleteSchema, InternalArtifactSchema } from '../../schemas'; -import { ArtifactConstants } from './common'; - -class ArtifactMigrationError extends Error { - constructor(message: string, public readonly meta?: unknown) { - super(message); - } -} - -const inflateAsync = promisify(_inflate); - -function isCompressed(artifact: InternalArtifactSchema) { - return artifact.compressionAlgorithm === 'zlib'; -} - -/** - * With v7.13, artifact storage was moved from a security_solution saved object to a fleet index - * in order to support Fleet Server. - */ -export const migrateArtifactsToFleet = async ( - soClient: SavedObjectsClient, - endpointArtifactClient: EndpointArtifactClientInterface, - logger: Logger -): Promise => { - let totalArtifactsMigrated = -1; - let hasMore = true; - - try { - while (hasMore) { - // Retrieve list of artifact records - const { saved_objects: artifactList, total } = - await soClient.find({ - type: ArtifactConstants.SAVED_OBJECT_TYPE, - page: 1, - perPage: 10, - }); - - if (totalArtifactsMigrated === -1) { - totalArtifactsMigrated = total; - if (total > 0) { - logger.info(`Migrating artifacts from SavedObject`); - } - } - - // If nothing else to process, then exit out - if (total === 0) { - hasMore = false; - if (totalArtifactsMigrated > 0) { - logger.info(`Total Artifacts migrated: ${totalArtifactsMigrated}`); - } - return; - } - - for (const artifact of artifactList) { - if (isCompressed(artifact.attributes)) { - artifact.attributes = { - ...artifact.attributes, - body: (await inflateAsync(Buffer.from(artifact.attributes.body, 'base64'))).toString( - 'base64' - ), - }; - } - - // Create new artifact in fleet index - await endpointArtifactClient.createArtifact(artifact.attributes); - // Delete old artifact from SO and if there are errors here, then ignore 404's - // since multiple kibana instances could be going at this - try { - await soClient.delete(ArtifactConstants.SAVED_OBJECT_TYPE, artifact.id); - } catch (e) { - if (e?.output?.statusCode !== 404) { - throw e; - } - logger.debug( - `Artifact Migration: Attempt to delete Artifact SO [${artifact.id}] returned 404` - ); - } - } - } - } catch (e) { - const error = new ArtifactMigrationError('Artifact SO migration failed', e); - logger.error(error); - throw error; - } -}; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts index a72d90250d0f5..efd6ef7ab719a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts @@ -13,39 +13,6 @@ import { migrations } from './migrations'; export const exceptionsArtifactSavedObjectType = ArtifactConstants.SAVED_OBJECT_TYPE; export const manifestSavedObjectType = ManifestConstants.SAVED_OBJECT_TYPE; -export const exceptionsArtifactSavedObjectMappings: SavedObjectsType['mappings'] = { - dynamic: false, - properties: { - identifier: { - type: 'keyword', - }, - compressionAlgorithm: { - type: 'keyword', - }, - encryptionAlgorithm: { - type: 'keyword', - }, - encodedSha256: { - type: 'keyword', - }, - encodedSize: { - type: 'long', - }, - decodedSha256: { - type: 'keyword', - }, - decodedSize: { - type: 'long', - }, - created: { - type: 'date', - }, - body: { - type: 'binary', - }, - }, -}; - export const manifestSavedObjectMappings: SavedObjectsType['mappings'] = { dynamic: false, properties: { @@ -72,13 +39,6 @@ export const manifestSavedObjectMappings: SavedObjectsType['mappings'] = { }, }; -export const exceptionsArtifactType: SavedObjectsType = { - name: exceptionsArtifactSavedObjectType, - hidden: false, - namespaceType: 'agnostic', - mappings: exceptionsArtifactSavedObjectMappings, -}; - export const manifestType: SavedObjectsType = { name: manifestSavedObjectType, hidden: false, diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 2b6048c122f27..fce56379f1b2e 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -69,7 +69,6 @@ import type { ITelemetryReceiver } from './lib/telemetry/receiver'; import { TelemetryReceiver } from './lib/telemetry/receiver'; import { licenseService } from './lib/license'; import { PolicyWatcher } from './endpoint/lib/policy/license_watch'; -import { migrateArtifactsToFleet } from './endpoint/lib/artifacts/migrate_artifacts_to_fleet'; import previewPolicy from './lib/detection_engine/routes/index/preview_policy.json'; import { createRuleExecutionLogService } from './lib/detection_engine/rule_monitoring'; import { getKibanaPrivilegesFeaturePrivileges, getCasesKibanaFeature } from './features'; @@ -448,17 +447,15 @@ export class Plugin implements ISecuritySolutionPlugin { // Migrate artifacts to fleet and then start the minifest task after that is done plugins.fleet.fleetSetupCompleted().then(() => { - migrateArtifactsToFleet(savedObjectsClient, artifactClient, logger).finally(() => { - logger.info('Dependent plugin setup complete - Starting ManifestTask'); - - if (this.manifestTask) { - this.manifestTask.start({ - taskManager, - }); - } else { - logger.error(new Error('User artifacts task not available.')); - } - }); + logger.info('Dependent plugin setup complete - Starting ManifestTask'); + + if (this.manifestTask) { + this.manifestTask.start({ + taskManager, + }); + } else { + logger.error(new Error('User artifacts task not available.')); + } }); // License related start diff --git a/x-pack/plugins/security_solution/server/saved_objects.ts b/x-pack/plugins/security_solution/server/saved_objects.ts index 394cab7f52455..d29e560ec64d1 100644 --- a/x-pack/plugins/security_solution/server/saved_objects.ts +++ b/x-pack/plugins/security_solution/server/saved_objects.ts @@ -13,7 +13,6 @@ import { legacyType as legacyRuleActionsType } from './lib/detection_engine/rule import { prebuiltRuleAssetType } from './lib/detection_engine/prebuilt_rules'; import { type as signalsMigrationType } from './lib/detection_engine/migrations/saved_objects'; import { - exceptionsArtifactType, manifestType, } from './endpoint/lib/artifacts/saved_object_mappings'; @@ -23,7 +22,6 @@ const types = [ legacyRuleActionsType, prebuiltRuleAssetType, timelineType, - exceptionsArtifactType, manifestType, signalsMigrationType, ];