[Fleet] Add ?full option to get package info endpoint to return all…

… package fields (#144343)

* add source mode to archive parsing

* add ?full patram to get pkg info

* revert archive change

* do not cache EPR fetch info

* add test package for source_mode

* add int test

* add new param to frontent

* use released package for test

* tidy for PR

* Add API docs

* add get all packages script

* get archive from cache even if not installed

* fix types

x-pack/plugins/fleet/common/openapi/bundled.json

@@ -667,6 +667,14 @@
           "name": "ignoreUnverified",
           "description": "Ignore if the package is fails signature verification",
           "in": "query"
+        },
+        {
+          "schema": {
+            "type": "boolean"
+          },
+          "name": "full",
+          "description": "Return all fields from the package manifest, not just those supported by the Elastic Package Registry",
+          "in": "query"
         }
       ],
       "post": {

x-pack/plugins/fleet/common/openapi/bundled.yaml

@@ -419,6 +419,13 @@ paths:
         name: ignoreUnverified
         description: Ignore if the package is fails signature verification
         in: query
+      - schema:
+          type: boolean
+        name: full
+        description: >-
+          Return all fields from the package manifest, not just those supported
+          by the Elastic Package Registry
+        in: query
     post:
       summary: Packages - Install
       tags: []

x-pack/plugins/fleet/common/openapi/paths/epm@packages@{pkg_name}@{pkg_version}.yaml

@@ -52,6 +52,11 @@ parameters:
     name: ignoreUnverified
     description: 'Ignore if the package is fails signature verification'
     in: query
+  - schema:
+      type: boolean
+    name: full
+    description: 'Return all fields from the package manifest, not just those supported by the Elastic Package Registry'
+    in: query
 post:
   summary: Packages - Install
   tags: []

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/multi_page_layout/index.tsx

@@ -70,7 +70,7 @@ export const CreatePackagePolicyMultiPage: CreatePackagePolicyParams = ({
     data: packageInfoData,
     error: packageInfoError,
     isLoading: isPackageInfoLoading,
-  } = useGetPackageInfoByKey(pkgName, pkgVersion, { prerelease: true });
+  } = useGetPackageInfoByKey(pkgName, pkgVersion, { prerelease: true, full: true });
 
   const {
     agentPolicy,

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/index.tsx

@@ -99,7 +99,7 @@ export const CreatePackagePolicySinglePage: CreatePackagePolicyParams = ({
     data: packageInfoData,
     error: packageInfoError,
     isLoading: isPackageInfoLoading,
-  } = useGetPackageInfoByKey(pkgName, pkgVersion, { prerelease: true });
+  } = useGetPackageInfoByKey(pkgName, pkgVersion, { full: true, prerelease: true });
   const packageInfo = useMemo(() => {
     if (packageInfoData && packageInfoData.item) {
       return packageInfoData.item;

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/index.tsx

@@ -266,7 +266,7 @@ export const EditPackagePolicyForm = memo<{
             const { data: packageData } = await sendGetPackageInfoByKey(
               _packageInfo!.name,
               _packageInfo!.version,
-              { prerelease: true }
+              { prerelease: true, full: true }
             );
 
             if (packageData?.item) {

x-pack/plugins/fleet/public/hooks/use_request/epm.ts

@@ -85,6 +85,7 @@ export const useGetPackageInfoByKey = (
   options?: {
     ignoreUnverified?: boolean;
     prerelease?: boolean;
+    full?: boolean;
   }
 ) => {
   const confirmOpenUnverified = useConfirmOpenUnverified();
@@ -96,7 +97,7 @@ export const useGetPackageInfoByKey = (
     method: 'get',
     query: {
       ...options,
-      ...(ignoreUnverifiedQueryParam ? { ignoreUnverified: ignoreUnverifiedQueryParam } : {}),
+      ...(ignoreUnverifiedQueryParam && { ignoreUnverified: ignoreUnverifiedQueryParam }),
     },
   });
 
@@ -130,6 +131,7 @@ export const sendGetPackageInfoByKey = (
   options?: {
     ignoreUnverified?: boolean;
     prerelease?: boolean;
+    full?: boolean;
   }
 ) => {
   return sendRequest<GetInfoResponse>({

x-pack/plugins/fleet/scripts/get_all_packages/get_all_packages.ts

@@ -0,0 +1,128 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import fetch from 'node-fetch';
+import { kibanaPackageJson } from '@kbn/utils';
+import { ToolingLog } from '@kbn/tooling-log';
+import { chunk } from 'lodash';
+
+import yargs from 'yargs/yargs';
+
+import type { PackageInfo } from '../../common';
+
+const REGISTRY_URL = 'https://epr-snapshot.elastic.co';
+const KIBANA_URL = 'http://localhost:5601';
+const KIBANA_USERNAME = 'elastic';
+const KIBANA_PASSWORD = 'changeme';
+const KIBANA_VERSION = kibanaPackageJson.version;
+
+const { base = '', prerelease = false, batchSize = 1 } = yargs(process.argv).argv;
+
+const logger = new ToolingLog({
+  level: 'info',
+  writeTo: process.stdout,
+});
+
+interface Result {
+  pkg: string;
+  epr: number;
+  archive: number;
+  archiveCached: number;
+}
+async function getPackage(name: string, version: string, full: boolean = false) {
+  const start = Date.now();
+  const res = await fetch(
+    `${KIBANA_URL}${base}/api/fleet/epm/packages/${name}/${version}?prerelease=true${
+      full ? '&full=true' : ''
+    }`,
+    {
+      headers: {
+        accept: '*/*',
+        'content-type': 'application/json',
+        'kbn-xsrf': 'xyz',
+        Authorization:
+          'Basic ' + Buffer.from(`${KIBANA_USERNAME}:${KIBANA_PASSWORD}`).toString('base64'),
+      },
+      method: 'GET',
+    }
+  );
+  const end = Date.now();
+
+  let body;
+  try {
+    body = await res.json();
+  } catch (e) {
+    logger.error(`Error parsing response: ${e}`);
+    throw e;
+  }
+
+  if (body.item && body.item.name) {
+    return { pkg: body.item, status: body.status, took: (end - start) / 1000 };
+  }
+
+  throw new Error(`Invalid package returned for ${name}-${version} : ${JSON.stringify(res)}`);
+}
+
+async function getAllPackages() {
+  const res = await fetch(
+    `${REGISTRY_URL}/search?kibana.version=${KIBANA_VERSION}${
+      prerelease ? '&prerelease=true' : ''
+    }`,
+    {
+      headers: {
+        accept: '*/*',
+      },
+      method: 'GET',
+    }
+  );
+  const body = await res.json();
+  return body as PackageInfo[];
+}
+
+async function performTest({ name, version }: { name: string; version: string }): Promise<Result> {
+  const eprResult = await getPackage(name, version);
+  const archiveResult = await getPackage(name, version, true);
+  const cachedArchiveResult = await getPackage(name, version, true);
+  logger.info(`✅ ${name}-${version}`);
+
+  return {
+    pkg: `${name}-${version}`,
+    epr: eprResult.took,
+    archive: archiveResult.took,
+    archiveCached: cachedArchiveResult.took,
+  };
+}
+
+export async function run() {
+  const allPackages = await getAllPackages();
+
+  const batches = chunk(allPackages, batchSize as number);
+  let allResults: Result[] = [];
+
+  const start = Date.now();
+  for (const batch of batches) {
+    const results = await Promise.all(batch.map(performTest));
+    allResults = [...allResults, ...(results.filter((v) => v) as Result[])];
+  }
+  const end = Date.now();
+  const took = (end - start) / 1000;
+  allResults.sort((a, b) => b.archive - a.archive);
+  logger.info(`Took ${took} seconds to get ${allResults.length} packages`);
+  logger.info(
+    'Average EPM time: ' + allResults.reduce((acc, { epr }) => acc + epr, 0) / allResults.length
+  );
+  logger.info(
+    'Average Archive time: ' +
+      allResults.reduce((acc, { archive }) => acc + archive, 0) / allResults.length
+  );
+  logger.info(
+    'Average Cache time: ' +
+      allResults.reduce((acc, { archiveCached }) => acc + archiveCached, 0) / allResults.length
+  );
+  // eslint-disable-next-line no-console
+  console.table(allResults);
+}

x-pack/plugins/fleet/scripts/get_all_packages/index.js

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+require('../../../../../src/setup_node_env');
+require('./get_all_packages').run();

x-pack/plugins/fleet/server/routes/epm/handlers.ts

@@ -210,15 +210,15 @@ export const getInfoHandler: FleetRequestHandler<
   try {
     const savedObjectsClient = (await context.fleet).epm.internalSoClient;
     const { pkgName, pkgVersion } = request.params;
-    const { ignoreUnverified = false, prerelease } = request.query;
+    const { ignoreUnverified = false, full = false, prerelease } = request.query;
     if (pkgVersion && !semverValid(pkgVersion)) {
       throw new FleetError('Package version is not a valid semver');
     }
     const res = await getPackageInfo({
       savedObjectsClient,
       pkgName,
       pkgVersion: pkgVersion || '',
-      skipArchive: true,
+      skipArchive: !full,
       ignoreUnverified,
       prerelease,
     });

x-pack/plugins/fleet/server/services/epm/archive/cache.ts

@@ -67,6 +67,11 @@ export const getArchivePackage = (args: SharedKey) => {
   };
 };
 
+/*
+ * This cache should only be used to store "full" package info generated from the package archive.
+ * NOT package info from the EPR API. This is because we parse extra fields from the archive
+ * which are not provided by the registry API.
+ */
 export const setPackageInfo = ({
   name,
   version,

x-pack/plugins/fleet/server/services/epm/packages/get.ts

@@ -300,8 +300,14 @@ export async function getPackageFromSource(options: {
       }
     }
   } else {
-    res = await Registry.getPackage(pkgName, pkgVersion, { ignoreUnverified });
-    logger.debug(`retrieved package ${pkgName}-${pkgVersion} from registry`);
+    res = getArchivePackage({ name: pkgName, version: pkgVersion });
+
+    if (res) {
+      logger.debug(`retrieved package ${pkgName}-${pkgVersion} from cache`);
+    } else {
+      res = await Registry.getPackage(pkgName, pkgVersion, { ignoreUnverified });
+      logger.debug(`retrieved package ${pkgName}-${pkgVersion} from registry`);
+    }
   }
   if (!res) {
     throw new FleetError(`package info for ${pkgName}-${pkgVersion} does not exist`);

x-pack/plugins/fleet/server/services/epm/registry/index.ts

@@ -251,14 +251,7 @@ export async function fetchCategories(
 
 export async function getInfo(name: string, version: string) {
   return withPackageSpan('Fetch package info', async () => {
-    let packageInfo = getPackageInfo({ name, version });
-    if (!packageInfo) {
-      packageInfo = await fetchInfo(name, version);
-      // only cache registry pkg info for integration pkgs because
-      // input type packages must get their pkg info from the archive
-      if (packageInfo.type === 'integration') setPackageInfo({ name, version, packageInfo });
-    }
-
+    const packageInfo = await fetchInfo(name, version);
     return packageInfo as RegistryPackage;
   });
 }
@@ -272,15 +265,12 @@ async function getPackageInfoFromArchiveOrCache(
   archivePath: string
 ): Promise<ArchivePackage> {
   const cachedInfo = getPackageInfo({ name, version });
-
   if (!cachedInfo) {
     const { packageInfo } = await generatePackageInfoFromArchiveBuffer(
       archiveBuffer,
       ensureContentType(archivePath)
     );
-    // set the download URL as it isn't contained in the manifest
-    // this allows us to re-download the archive during package install
-    setPackageInfo({ packageInfo: { ...packageInfo, download: archivePath }, name, version });
+    setPackageInfo({ packageInfo, name, version });
     return packageInfo;
   } else {
     return cachedInfo;

x-pack/plugins/fleet/server/types/rest_spec/epm.ts

@@ -46,6 +46,7 @@ export const GetInfoRequestSchema = {
   query: schema.object({
     ignoreUnverified: schema.maybe(schema.boolean()),
     prerelease: schema.maybe(schema.boolean()),
+    full: schema.maybe(schema.boolean()),
   }),
 };
 

x-pack/test/fleet_api_integration/apis/epm/get.ts

@@ -6,6 +6,7 @@
  */
 
 import expect from '@kbn/expect';
+import { PackageInfo } from '@kbn/fleet-plugin/common/types/models/epm';
 import fs from 'fs';
 import path from 'path';
 import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
@@ -172,5 +173,40 @@ export default function (providerContext: FtrProviderContext) {
           .expect(200);
       });
     });
+    it('returns package info from the archive if ?full=true', async function () {
+      const res = await supertest
+        .get(`/api/fleet/epm/packages/non_epr_fields/1.0.0?full=true`)
+        .expect(200);
+      const packageInfo = res.body.item as PackageInfo;
+      expect(packageInfo?.data_streams?.length).equal(3);
+      const dataStream = packageInfo?.data_streams?.find(
+        ({ dataset }) => dataset === 'non_epr_fields.test_metrics_2'
+      );
+      expect(dataStream?.elasticsearch?.source_mode).equal('default');
+    });
+    it('returns package info from the registry if ?full=false', async function () {
+      const res = await supertest
+        .get(`/api/fleet/epm/packages/non_epr_fields/1.0.0?full=false`)
+        .expect(200);
+      const packageInfo = res.body.item as PackageInfo;
+      expect(packageInfo?.data_streams?.length).equal(3);
+      const dataStream = packageInfo?.data_streams?.find(
+        ({ dataset }) => dataset === 'non_epr_fields.test_metrics_2'
+      );
+      // this field is only returned if we go to the archive
+      // it is not part of the EPR API
+      expect(dataStream?.elasticsearch?.source_mode).equal(undefined);
+    });
+    it('returns package info from the registry if ?full not provided', async function () {
+      const res = await supertest
+        .get(`/api/fleet/epm/packages/non_epr_fields/1.0.0?full=false`)
+        .expect(200);
+      const packageInfo = res.body.item as PackageInfo;
+      expect(packageInfo?.data_streams?.length).equal(3);
+      const dataStream = packageInfo?.data_streams?.find(
+        ({ dataset }) => dataset === 'non_epr_fields.test_metrics_2'
+      );
+      expect(dataStream?.elasticsearch?.source_mode).equal(undefined);
+    });
   });
 }

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/1.0.0/data_stream/test_logs/fields/ecs.yml

@@ -0,0 +1,6 @@
+- name: logs_test_name
+  title: logs_test_title
+  type: text
+- name: new_field_name
+  title: new_field_title
+  type: keyword

x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/1.0.0/data_stream/test_logs/fields/fields.yml

@@ -0,0 +1,16 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: >
+    Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: >
+    Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: >
+    Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: >
+    Event timestamp.