[Security solution][Endpoint] Removes zip compression when creating a…

…rtifacts (elastic#101379)

* Removes zlib compression when creating artifacts. Also fixes related unit tests and removes old code

* Replaces artifact in new manifest using the ones from fleet client with zlip compression

* Fixes create_policy_artifact_manifest pushArtifacts missing new manifest. Also fixes unit tests

Co-authored-by: Kibana Machine <[email protected]>

x-pack/plugins/security_solution/common/endpoint/schema/common.ts

@@ -15,6 +15,7 @@ export type CompressionAlgorithm = t.TypeOf<typeof compressionAlgorithm>;
 
 export const compressionAlgorithmDispatch = t.keyof({
   zlib: null,
+  none: null,
 });
 export type CompressionAlgorithmDispatch = t.TypeOf<typeof compressionAlgorithmDispatch>;
 

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts

@@ -6,7 +6,6 @@
  */
 
 import { createHash } from 'crypto';
-import { deflate } from 'zlib';
 import type {
   Entry,
   EntryNested,
@@ -21,9 +20,7 @@ import {
 } from '@kbn/securitysolution-list-constants';
 import { ExceptionListClient } from '../../../../../lists/server';
 import {
-  internalArtifactCompleteSchema,
   InternalArtifactCompleteSchema,
-  InternalArtifactSchema,
   TranslatedEntry,
   translatedEntry as translatedEntryType,
   translatedEntryMatchAnyMatcher,
@@ -60,28 +57,6 @@ export async function buildArtifact(
   };
 }
 
-export async function maybeCompressArtifact(
-  uncompressedArtifact: InternalArtifactSchema
-): Promise<InternalArtifactSchema> {
-  const compressedArtifact = { ...uncompressedArtifact };
-  if (internalArtifactCompleteSchema.is(uncompressedArtifact)) {
-    const compressedArtifactBody = await compressExceptionList(
-      Buffer.from(uncompressedArtifact.body, 'base64')
-    );
-    compressedArtifact.body = compressedArtifactBody.toString('base64');
-    compressedArtifact.encodedSize = compressedArtifactBody.byteLength;
-    compressedArtifact.compressionAlgorithm = 'zlib';
-    compressedArtifact.encodedSha256 = createHash('sha256')
-      .update(compressedArtifactBody)
-      .digest('hex');
-  }
-  return compressedArtifact;
-}
-
-export function isCompressed(artifact: InternalArtifactSchema) {
-  return artifact.compressionAlgorithm === 'zlib';
-}
-
 export async function getFilteredEndpointExceptionList(
   eClient: ExceptionListClient,
   schemaVersion: string,
@@ -297,15 +272,3 @@ function translateEntry(
     }
   }
 }
-
-export async function compressExceptionList(buffer: Buffer): Promise<Buffer> {
-  return new Promise((resolve, reject) => {
-    deflate(buffer, function (err, buf) {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(buf);
-      }
-    });
-  });
-}

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts

@@ -37,8 +37,8 @@ describe('manifest', () => {
   let ARTIFACT_COPY_TRUSTED_APPS_WINDOWS: InternalArtifactCompleteSchema;
 
   beforeAll(async () => {
-    ARTIFACTS = await getMockArtifacts({ compress: true });
-    ARTIFACTS_COPY = await getMockArtifacts({ compress: true });
+    ARTIFACTS = await getMockArtifacts();
+    ARTIFACTS_COPY = await getMockArtifacts();
     ARTIFACT_EXCEPTIONS_MACOS = ARTIFACTS[0];
     ARTIFACT_EXCEPTIONS_WINDOWS = ARTIFACTS[1];
     ARTIFACT_EXCEPTIONS_LINUX = ARTIFACTS[2];

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts

@@ -12,6 +12,7 @@ import {
   InternalArtifactSchema,
   InternalManifestSchema,
   InternalManifestEntrySchema,
+  InternalArtifactCompleteSchema,
 } from '../../schemas/artifacts';
 import {
   ManifestSchemaVersion,
@@ -139,6 +140,27 @@ export class Manifest {
     return this.allEntries.get(getArtifactId(artifact))?.specificTargetPolicies;
   }
 
+  /**
+   * Replaces an artifact from all the collections.
+   *
+   * @param artifact An InternalArtifactCompleteSchema representing the artifact.
+   */
+  public replaceArtifact(artifact: InternalArtifactCompleteSchema) {
+    const existingEntry = this.allEntries.get(getArtifactId(artifact));
+    if (existingEntry) {
+      existingEntry.entry = new ManifestEntry(artifact);
+
+      this.allEntries.set(getArtifactId(artifact), existingEntry);
+      this.defaultEntries.set(getArtifactId(artifact), existingEntry.entry);
+
+      existingEntry.specificTargetPolicies.forEach((policyId) => {
+        const entries = this.policySpecificEntries.get(policyId) || new Map();
+        entries.set(existingEntry.entry.getDocId(), existingEntry.entry);
+        this.policySpecificEntries.set(policyId, entries);
+      });
+    }
+  }
+
   public diff(manifest: Manifest): ManifestDiff {
     const diff: ManifestDiff = {
       additions: [],

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts

@@ -15,7 +15,7 @@ describe('manifest_entry', () => {
     let manifestEntry: ManifestEntry;
 
     beforeAll(async () => {
-      artifact = await getInternalArtifactMock('windows', 'v1', { compress: true });
+      artifact = await getInternalArtifactMock('windows', 'v1');
       manifestEntry = new ManifestEntry(artifact);
     });
 
@@ -35,15 +35,15 @@ describe('manifest_entry', () => {
 
     test('Correct sha256 is returned', () => {
       expect(manifestEntry.getEncodedSha256()).toEqual(
-        '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e'
+        '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3'
       );
       expect(manifestEntry.getDecodedSha256()).toEqual(
         '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3'
       );
     });
 
     test('Correct size is returned', () => {
-      expect(manifestEntry.getEncodedSize()).toEqual(147);
+      expect(manifestEntry.getEncodedSize()).toEqual(432);
       expect(manifestEntry.getDecodedSize()).toEqual(432);
     });
 
@@ -59,12 +59,12 @@ describe('manifest_entry', () => {
 
     test('Correct record is returned', () => {
       expect(manifestEntry.getRecord()).toEqual({
-        compression_algorithm: 'zlib',
+        compression_algorithm: 'none',
         encryption_algorithm: 'none',
         decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3',
-        encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e',
+        encoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3',
         decoded_size: 432,
-        encoded_size: 147,
+        encoded_size: 432,
         relative_url:
           '/api/fleet/artifacts/endpoint-exceptionlist-windows-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3',
       });

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrate_artifacts_to_fleet.test.ts

@@ -49,7 +49,7 @@ describe('When migrating artifacts to fleet', () => {
           type: '',
           id: 'abc123',
           references: [],
-          attributes: await getInternalArtifactMock('windows', 'v1', { compress: true }),
+          attributes: await getInternalArtifactMock('windows', 'v1'),
         },
       ])
     );

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/mocks.ts

@@ -17,49 +17,49 @@ import {
 import { ArtifactConstants } from './common';
 import { Manifest } from './manifest';
 
-export const getMockArtifacts = async (opts?: { compress: boolean }) => {
+export const getMockArtifacts = async () => {
   return Promise.all([
     // Exceptions items
     ...ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.map<Promise<InternalArtifactCompleteSchema>>(
       async (os) => {
-        return getInternalArtifactMock(os, 'v1', opts);
+        return getInternalArtifactMock(os, 'v1');
       }
     ),
     // Trusted Apps items
     ...ArtifactConstants.SUPPORTED_TRUSTED_APPS_OPERATING_SYSTEMS.map<
       Promise<InternalArtifactCompleteSchema>
     >(async (os) => {
-      return getInternalArtifactMock(os, 'v1', opts, ArtifactConstants.GLOBAL_TRUSTED_APPS_NAME);
+      return getInternalArtifactMock(os, 'v1', ArtifactConstants.GLOBAL_TRUSTED_APPS_NAME);
     }),
   ]);
 };
 
-export const getMockArtifactsWithDiff = async (opts?: { compress: boolean }) => {
+export const getMockArtifactsWithDiff = async () => {
   return Promise.all(
     ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.map<Promise<InternalArtifactCompleteSchema>>(
       async (os) => {
         if (os === 'macos') {
           return getInternalArtifactMockWithDiffs(os, 'v1');
         }
-        return getInternalArtifactMock(os, 'v1', opts);
+        return getInternalArtifactMock(os, 'v1');
       }
     )
   );
 };
 
-export const getEmptyMockArtifacts = async (opts?: { compress: boolean }) => {
+export const getEmptyMockArtifacts = async () => {
   return Promise.all(
     ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.map<Promise<InternalArtifactCompleteSchema>>(
       async (os) => {
-        return getEmptyInternalArtifactMock(os, 'v1', opts);
+        return getEmptyInternalArtifactMock(os, 'v1');
       }
     )
   );
 };
 
-export const getMockManifest = async (opts?: { compress: boolean }) => {
+export const getMockManifest = async () => {
   const manifest = new Manifest();
-  const artifacts = await getMockArtifacts(opts);
+  const artifacts = await getMockArtifacts();
   artifacts.forEach((artifact) => manifest.addEntry(artifact));
   return manifest;
 };

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts

@@ -106,7 +106,7 @@ describe('task', () => {
     let ARTIFACT_TRUSTED_APPS_MACOS: InternalArtifactCompleteSchema;
 
     beforeAll(async () => {
-      const artifacts = await getMockArtifacts({ compress: true });
+      const artifacts = await getMockArtifacts();
       ARTIFACT_EXCEPTIONS_MACOS = artifacts[0];
       ARTIFACT_EXCEPTIONS_WINDOWS = artifacts[1];
       ARTIFACT_TRUSTED_APPS_MACOS = artifacts[2];
@@ -167,7 +167,7 @@ describe('task', () => {
 
       expect(manifestManager.getLastComputedManifest).toHaveBeenCalled();
       expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(lastManifest);
-      expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([]);
+      expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([], newManifest);
       expect(manifestManager.commit).not.toHaveBeenCalled();
       expect(manifestManager.tryDispatch).toHaveBeenCalledWith(newManifest);
       expect(manifestManager.deleteArtifacts).toHaveBeenCalledWith([]);
@@ -192,10 +192,10 @@ describe('task', () => {
 
       expect(manifestManager.getLastComputedManifest).toHaveBeenCalled();
       expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(lastManifest);
-      expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([
-        ARTIFACT_EXCEPTIONS_MACOS,
-        ARTIFACT_TRUSTED_APPS_MACOS,
-      ]);
+      expect(manifestManager.pushArtifacts).toHaveBeenCalledWith(
+        [ARTIFACT_EXCEPTIONS_MACOS, ARTIFACT_TRUSTED_APPS_MACOS],
+        newManifest
+      );
       expect(manifestManager.commit).not.toHaveBeenCalled();
       expect(manifestManager.tryDispatch).not.toHaveBeenCalled();
       expect(manifestManager.deleteArtifacts).not.toHaveBeenCalled();
@@ -221,10 +221,10 @@ describe('task', () => {
 
       expect(manifestManager.getLastComputedManifest).toHaveBeenCalled();
       expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(lastManifest);
-      expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([
-        ARTIFACT_EXCEPTIONS_MACOS,
-        ARTIFACT_TRUSTED_APPS_MACOS,
-      ]);
+      expect(manifestManager.pushArtifacts).toHaveBeenCalledWith(
+        [ARTIFACT_EXCEPTIONS_MACOS, ARTIFACT_TRUSTED_APPS_MACOS],
+        newManifest
+      );
       expect(manifestManager.commit).toHaveBeenCalledWith(newManifest);
       expect(manifestManager.tryDispatch).not.toHaveBeenCalled();
       expect(manifestManager.deleteArtifacts).not.toHaveBeenCalled();
@@ -251,10 +251,10 @@ describe('task', () => {
 
       expect(manifestManager.getLastComputedManifest).toHaveBeenCalled();
       expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(lastManifest);
-      expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([
-        ARTIFACT_EXCEPTIONS_MACOS,
-        ARTIFACT_TRUSTED_APPS_MACOS,
-      ]);
+      expect(manifestManager.pushArtifacts).toHaveBeenCalledWith(
+        [ARTIFACT_EXCEPTIONS_MACOS, ARTIFACT_TRUSTED_APPS_MACOS],
+        newManifest
+      );
       expect(manifestManager.commit).toHaveBeenCalledWith(newManifest);
       expect(manifestManager.tryDispatch).toHaveBeenCalledWith(newManifest);
       expect(manifestManager.deleteArtifacts).not.toHaveBeenCalled();
@@ -284,7 +284,10 @@ describe('task', () => {
 
       expect(manifestManager.getLastComputedManifest).toHaveBeenCalled();
       expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(lastManifest);
-      expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([ARTIFACT_TRUSTED_APPS_MACOS]);
+      expect(manifestManager.pushArtifacts).toHaveBeenCalledWith(
+        [ARTIFACT_TRUSTED_APPS_MACOS],
+        newManifest
+      );
       expect(manifestManager.commit).toHaveBeenCalledWith(newManifest);
       expect(manifestManager.tryDispatch).toHaveBeenCalledWith(newManifest);
       expect(manifestManager.deleteArtifacts).toHaveBeenCalledWith([ARTIFACT_ID_1]);
@@ -314,7 +317,7 @@ describe('task', () => {
 
       expect(manifestManager.getLastComputedManifest).toHaveBeenCalled();
       expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(lastManifest);
-      expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([]);
+      expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([], newManifest);
       expect(manifestManager.commit).toHaveBeenCalledWith(newManifest);
       expect(manifestManager.tryDispatch).toHaveBeenCalledWith(newManifest);
       expect(manifestManager.deleteArtifacts).toHaveBeenCalledWith([]);

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts

@@ -130,7 +130,8 @@ export class ManifestTask {
       const diff = newManifest.diff(oldManifest);
 
       const persistErrors = await manifestManager.pushArtifacts(
-        diff.additions as InternalArtifactCompleteSchema[]
+        diff.additions as InternalArtifactCompleteSchema[],
+        newManifest
       );
       if (persistErrors.length) {
         reportErrors(this.logger, persistErrors);

x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts

@@ -5,33 +5,13 @@
  * 2.0.
  */
 
-import {
-  buildArtifact,
-  maybeCompressArtifact,
-  isCompressed,
-  ArtifactConstants,
-} from '../../lib/artifacts';
+import { buildArtifact, ArtifactConstants } from '../../lib/artifacts';
 import { getTranslatedExceptionListMock } from './lists.mock';
-import {
-  InternalManifestSchema,
-  internalArtifactCompleteSchema,
-  InternalArtifactCompleteSchema,
-} from './saved_objects';
-
-const compressArtifact = async (artifact: InternalArtifactCompleteSchema) => {
-  const compressedArtifact = await maybeCompressArtifact(artifact);
-  if (!isCompressed(compressedArtifact)) {
-    throw new Error(`Unable to compress artifact: ${artifact.identifier}`);
-  } else if (!internalArtifactCompleteSchema.is(compressedArtifact)) {
-    throw new Error(`Incomplete artifact detected: ${artifact.identifier}`);
-  }
-  return compressedArtifact;
-};
+import { InternalManifestSchema, InternalArtifactCompleteSchema } from './saved_objects';
 
 export const getInternalArtifactMock = async (
   os: string,
   schemaVersion: string,
-  opts?: { compress: boolean },
   artifactName: string = ArtifactConstants.GLOBAL_ALLOWLIST_NAME
 ): Promise<InternalArtifactCompleteSchema> => {
   const artifact = await buildArtifact(
@@ -40,23 +20,21 @@ export const getInternalArtifactMock = async (
     os,
     artifactName
   );
-  return opts?.compress ? compressArtifact(artifact) : artifact;
+  return artifact;
 };
 
 export const getEmptyInternalArtifactMock = async (
   os: string,
   schemaVersion: string,
-  opts?: { compress: boolean },
   artifactName: string = ArtifactConstants.GLOBAL_ALLOWLIST_NAME
 ): Promise<InternalArtifactCompleteSchema> => {
   const artifact = await buildArtifact({ entries: [] }, schemaVersion, os, artifactName);
-  return opts?.compress ? compressArtifact(artifact) : artifact;
+  return artifact;
 };
 
 export const getInternalArtifactMockWithDiffs = async (
   os: string,
-  schemaVersion: string,
-  opts?: { compress: boolean }
+  schemaVersion: string
 ): Promise<InternalArtifactCompleteSchema> => {
   const mock = getTranslatedExceptionListMock();
   mock.entries.pop();
@@ -66,7 +44,7 @@ export const getInternalArtifactMockWithDiffs = async (
     os,
     ArtifactConstants.GLOBAL_ALLOWLIST_NAME
   );
-  return opts?.compress ? compressArtifact(artifact) : artifact;
+  return artifact;
 };
 
 export const getInternalManifestMock = (): InternalManifestSchema => ({