From 99695824592823c1cb4a845d27dbf5733100e389 Mon Sep 17 00:00:00 2001
From: Mark Hopkin <mark.hopkin@elastic.co>
Date: Tue, 1 Nov 2022 16:28:35 +0000
Subject: [PATCH 01/13] add source mode to archive parsing

---
 x-pack/plugins/fleet/common/types/models/epm.ts | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts
index 4ef4a7fdd751b..728c7f4364786 100644
--- a/x-pack/plugins/fleet/common/types/models/epm.ts
+++ b/x-pack/plugins/fleet/common/types/models/epm.ts
@@ -115,8 +115,9 @@ export type MonitoringType = typeof monitoringTypes;
 export type InstallablePackage = RegistryPackage | ArchivePackage;
 
 export type ArchivePackage = PackageSpecManifest &
-  // should an uploaded package be able to specify `internal`?
-  Pick<RegistryPackage, 'readme' | 'assets' | 'data_streams' | 'internal' | 'elasticsearch'>;
+  Pick<RegistryPackage, 'readme' | 'assets' | 'internal' | 'elasticsearch'> & {
+    data_streams?: ArchiveDataStream[];
+  };
 
 export interface BundledPackage {
   name: string;
@@ -346,6 +347,10 @@ export interface RegistryDataStream {
   [RegistryDataStreamKeys.dataset_is_prefix]?: boolean;
 }
 
+export type ArchiveDataStream = RegistryDataStream & {
+  [RegistryDataStreamKeys.elasticsearch]?: ArchiveElasticsearch;
+};
+
 export interface RegistryElasticsearch {
   privileges?: RegistryDataStreamPrivileges;
   'index_template.settings'?: estypes.IndicesIndexSettings;
@@ -354,6 +359,10 @@ export interface RegistryElasticsearch {
   source_mode?: 'default' | 'synthetic';
 }
 
+export type ArchiveElasticsearch = RegistryElasticsearch & {
+  source_mode?: 'default' | 'synthetic';
+};
+
 export interface RegistryDataStreamPrivileges {
   cluster?: string[];
   indices?: string[];

From 1e2157b7d6a2f06b5c0293379351408b16dbc121 Mon Sep 17 00:00:00 2001
From: Mark Hopkin <mark.hopkin@elastic.co>
Date: Tue, 1 Nov 2022 16:31:05 +0000
Subject: [PATCH 02/13] add ?full patram to get pkg info

---
 x-pack/plugins/fleet/server/routes/epm/handlers.ts | 4 ++--
 x-pack/plugins/fleet/server/types/rest_spec/epm.ts | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/x-pack/plugins/fleet/server/routes/epm/handlers.ts b/x-pack/plugins/fleet/server/routes/epm/handlers.ts
index 5ef609fe9b6cc..e66bfa473edbb 100644
--- a/x-pack/plugins/fleet/server/routes/epm/handlers.ts
+++ b/x-pack/plugins/fleet/server/routes/epm/handlers.ts
@@ -210,7 +210,7 @@ export const getInfoHandler: FleetRequestHandler<
   try {
     const savedObjectsClient = (await context.fleet).epm.internalSoClient;
     const { pkgName, pkgVersion } = request.params;
-    const { ignoreUnverified = false, prerelease } = request.query;
+    const { ignoreUnverified = false, full = false, prerelease } = request.query;
     if (pkgVersion && !semverValid(pkgVersion)) {
       throw new FleetError('Package version is not a valid semver');
     }
@@ -218,7 +218,7 @@ export const getInfoHandler: FleetRequestHandler<
       savedObjectsClient,
       pkgName,
       pkgVersion: pkgVersion || '',
-      skipArchive: true,
+      skipArchive: !full,
       ignoreUnverified,
       prerelease,
     });
diff --git a/x-pack/plugins/fleet/server/types/rest_spec/epm.ts b/x-pack/plugins/fleet/server/types/rest_spec/epm.ts
index 9e36413f80150..9dc11599fa0de 100644
--- a/x-pack/plugins/fleet/server/types/rest_spec/epm.ts
+++ b/x-pack/plugins/fleet/server/types/rest_spec/epm.ts
@@ -46,6 +46,7 @@ export const GetInfoRequestSchema = {
   query: schema.object({
     ignoreUnverified: schema.maybe(schema.boolean()),
     prerelease: schema.maybe(schema.boolean()),
+    full: schema.maybe(schema.boolean()),
   }),
 };
 

From 2d5af12f7f151fc58834e50ba60cae860af0c3a2 Mon Sep 17 00:00:00 2001
From: Mark Hopkin <mark.hopkin@elastic.co>
Date: Tue, 1 Nov 2022 16:34:09 +0000
Subject: [PATCH 03/13] revert archive change

---
 x-pack/plugins/fleet/common/types/models/epm.ts | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts
index 728c7f4364786..8341dcef581f1 100644
--- a/x-pack/plugins/fleet/common/types/models/epm.ts
+++ b/x-pack/plugins/fleet/common/types/models/epm.ts
@@ -115,9 +115,8 @@ export type MonitoringType = typeof monitoringTypes;
 export type InstallablePackage = RegistryPackage | ArchivePackage;
 
 export type ArchivePackage = PackageSpecManifest &
-  Pick<RegistryPackage, 'readme' | 'assets' | 'internal' | 'elasticsearch'> & {
-    data_streams?: ArchiveDataStream[];
-  };
+  // should an uploaded package be able to specify `internal`?
+  Pick<RegistryPackage, 'readme' | 'assets' | 'data_streams' | 'internal' | 'elasticsearch'>;
 
 export interface BundledPackage {
   name: string;
@@ -347,10 +346,6 @@ export interface RegistryDataStream {
   [RegistryDataStreamKeys.dataset_is_prefix]?: boolean;
 }
 
-export type ArchiveDataStream = RegistryDataStream & {
-  [RegistryDataStreamKeys.elasticsearch]?: ArchiveElasticsearch;
-};
-
 export interface RegistryElasticsearch {
   privileges?: RegistryDataStreamPrivileges;
   'index_template.settings'?: estypes.IndicesIndexSettings;

From 78dfc253aabd5dd44207b502a891c7d121868118 Mon Sep 17 00:00:00 2001
From: Mark Hopkin <mark.hopkin@elastic.co>
Date: Tue, 1 Nov 2022 16:34:45 +0000
Subject: [PATCH 04/13] do not cache EPR fetch info

---
 .../fleet/server/services/epm/registry/index.ts    | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)

diff --git a/x-pack/plugins/fleet/server/services/epm/registry/index.ts b/x-pack/plugins/fleet/server/services/epm/registry/index.ts
index a6259d1eb6552..3c24e817d798e 100644
--- a/x-pack/plugins/fleet/server/services/epm/registry/index.ts
+++ b/x-pack/plugins/fleet/server/services/epm/registry/index.ts
@@ -251,14 +251,7 @@ export async function fetchCategories(
 
 export async function getInfo(name: string, version: string) {
   return withPackageSpan('Fetch package info', async () => {
-    let packageInfo = getPackageInfo({ name, version });
-    if (!packageInfo) {
-      packageInfo = await fetchInfo(name, version);
-      // only cache registry pkg info for integration pkgs because
-      // input type packages must get their pkg info from the archive
-      if (packageInfo.type === 'integration') setPackageInfo({ name, version, packageInfo });
-    }
-
+    const packageInfo = await fetchInfo(name, version);
     return packageInfo as RegistryPackage;
   });
 }
@@ -272,15 +265,12 @@ async function getPackageInfoFromArchiveOrCache(
   archivePath: string
 ): Promise<ArchivePackage> {
   const cachedInfo = getPackageInfo({ name, version });
-
   if (!cachedInfo) {
     const { packageInfo } = await generatePackageInfoFromArchiveBuffer(
       archiveBuffer,
       ensureContentType(archivePath)
     );
-    // set the download URL as it isn't contained in the manifest
-    // this allows us to re-download the archive during package install
-    setPackageInfo({ packageInfo: { ...packageInfo, download: archivePath }, name, version });
+    setPackageInfo({ packageInfo, name, version });
     return packageInfo;
   } else {
     return cachedInfo;

From 187c3cf436bccfbc89de4f9c7baa462f57f26504 Mon Sep 17 00:00:00 2001
From: Mark Hopkin <mark.hopkin@elastic.co>
Date: Tue, 1 Nov 2022 16:35:23 +0000
Subject: [PATCH 05/13] add test package for source_mode

---
 .../data_stream/test_logs/fields/ecs.yml      |  6 +++
 .../data_stream/test_logs/fields/fields.yml   | 16 +++++++
 .../0.2.0/data_stream/test_logs/manifest.yml  | 13 ++++++
 .../data_stream/test_metrics/fields/ecs.yml   |  6 +++
 .../test_metrics/fields/fields.yml            | 16 +++++++
 .../data_stream/test_metrics/manifest.yml     | 16 +++++++
 .../data_stream/test_metrics_2/fields/ecs.yml |  6 +++
 .../test_metrics_2/fields/fields.yml          | 16 +++++++
 .../data_stream/test_metrics_2/manifest.yml   | 16 +++++++
 .../non_epr_fields/0.2.0/docs/README.md       |  3 ++
 .../non_epr_fields/0.2.0/manifest.yml         | 42 +++++++++++++++++++
 11 files changed, 156 insertions(+)
 create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_logs/fields/ecs.yml
 create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_logs/fields/fields.yml
 create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_logs/manifest.yml
 create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics/fields/ecs.yml
 create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics/fields/fields.yml
 create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics/manifest.yml
 create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics_2/fields/ecs.yml
 create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics_2/fields/fields.yml
 create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics_2/manifest.yml
 create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/docs/README.md
 create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/manifest.yml

diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_logs/fields/ecs.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_logs/fields/ecs.yml
new file mode 100644
index 0000000000000..7df52cc11fd20
--- /dev/null
+++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_logs/fields/ecs.yml
@@ -0,0 +1,6 @@
+- name: logs_test_name
+  title: logs_test_title
+  type: text
+- name: new_field_name
+  title: new_field_title
+  type: keyword
diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_logs/fields/fields.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_logs/fields/fields.yml
new file mode 100644
index 0000000000000..6e003ed0ad147
--- /dev/null
+++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_logs/fields/fields.yml
@@ -0,0 +1,16 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: >
+    Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: >
+    Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: >
+    Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: >
+    Event timestamp.
diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_logs/manifest.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_logs/manifest.yml
new file mode 100644
index 0000000000000..83a6c3079cf8f
--- /dev/null
+++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_logs/manifest.yml
@@ -0,0 +1,13 @@
+title: Test Dataset
+
+type: logs
+
+streams:
+  - input: test_input1
+    title: "test input 1"
+    vars:
+    - name: test_var_required
+      type: string
+      title: Test Var
+      required: false
+      show_user: true
\ No newline at end of file
diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics/fields/ecs.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics/fields/ecs.yml
new file mode 100644
index 0000000000000..8fb3ccd3de8fd
--- /dev/null
+++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics/fields/ecs.yml
@@ -0,0 +1,6 @@
+- name: metrics_test_name
+  title: metrics_test_title
+  type: keyword
+- name: metrics_test_name2
+  title: metrics_test_title2
+  type: keyword
\ No newline at end of file
diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics/fields/fields.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics/fields/fields.yml
new file mode 100644
index 0000000000000..6e003ed0ad147
--- /dev/null
+++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics/fields/fields.yml
@@ -0,0 +1,16 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: >
+    Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: >
+    Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: >
+    Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: >
+    Event timestamp.
diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics/manifest.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics/manifest.yml
new file mode 100644
index 0000000000000..701322f1b03cf
--- /dev/null
+++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics/manifest.yml
@@ -0,0 +1,16 @@
+title: Test Dataset
+
+type: metrics
+
+streams:
+  - input: test_input2
+    title: "test input 2"
+    vars:
+    - name: test_var_required
+      type: string
+      title: Test Var
+      required: false
+      show_user: true
+
+elasticsearch:
+  source_mode: "synthetic"
\ No newline at end of file
diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics_2/fields/ecs.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics_2/fields/ecs.yml
new file mode 100644
index 0000000000000..8fb3ccd3de8fd
--- /dev/null
+++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics_2/fields/ecs.yml
@@ -0,0 +1,6 @@
+- name: metrics_test_name
+  title: metrics_test_title
+  type: keyword
+- name: metrics_test_name2
+  title: metrics_test_title2
+  type: keyword
\ No newline at end of file
diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics_2/fields/fields.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics_2/fields/fields.yml
new file mode 100644
index 0000000000000..6e003ed0ad147
--- /dev/null
+++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics_2/fields/fields.yml
@@ -0,0 +1,16 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: >
+    Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: >
+    Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: >
+    Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: >
+    Event timestamp.
diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics_2/manifest.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics_2/manifest.yml
new file mode 100644
index 0000000000000..e51fec7e5a450
--- /dev/null
+++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics_2/manifest.yml
@@ -0,0 +1,16 @@
+title: Test Dataset
+
+type: metrics
+
+streams:
+  - input: test_input3
+    title: "test input 3"
+    vars:
+    - name: test_var_required
+      type: string
+      title: Test Var
+      required: false
+      show_user: true
+
+elasticsearch:
+  source_mode: "default"
\ No newline at end of file
diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/docs/README.md b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/docs/README.md
new file mode 100644
index 0000000000000..1e93700e2fffd
--- /dev/null
+++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/docs/README.md
@@ -0,0 +1,3 @@
+# Test package
+
+For testing getting package info from the archive as opposed to EPR. This package contains fields that EPR does not return.
diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/manifest.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/manifest.yml
new file mode 100644
index 0000000000000..29b5dc60149c8
--- /dev/null
+++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/manifest.yml
@@ -0,0 +1,42 @@
+format_version: 1.0.0
+name: non_epr_fields
+title: Non EPR Fields
+description: This package contain fields in its manifest and datastreams such as source_mode which are not returned by the EPR API.
+version: 0.2.0
+categories: []
+release: beta
+type: integration
+license: basic
+owner:
+  github: elastic/fleet
+
+requirement:
+  elasticsearch:
+    versions: '>7.7.0'
+  kibana:
+    versions: '>7.7.0'
+policy_templates:
+  - name: p1
+    title: Synthetic source undefined
+    description: description
+    inputs:
+      - type: test_input1
+        title: source_mode undefined - defaults to unchecked, editable
+        description: source_mode undefined
+        enabled: true
+  - name: p2
+    title: Synthetic source synthetic
+    description: description
+    inputs:
+      - type: test_input2
+        title: source_mode 'synthetic' - default to checked, editable
+        description: source_mode synthetic should default to true and be editable
+        enabled: true
+  - name: p1
+    title: Synthetic source default
+    description: description
+    inputs:
+      - type: test_input3
+        title: source_mode 'default' - default to unchecked, not editable
+        description: source_mode default defaults to disabled and is not editable
+        enabled: true
\ No newline at end of file

From c3507885e795475d58ea87f290decf83483b85b7 Mon Sep 17 00:00:00 2001
From: Mark Hopkin <mark.hopkin@elastic.co>
Date: Tue, 1 Nov 2022 16:46:00 +0000
Subject: [PATCH 06/13] add int test

---
 .../fleet_api_integration/apis/epm/get.ts     | 38 +++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/x-pack/test/fleet_api_integration/apis/epm/get.ts b/x-pack/test/fleet_api_integration/apis/epm/get.ts
index 17f83b0b3c534..4834559ec2eac 100644
--- a/x-pack/test/fleet_api_integration/apis/epm/get.ts
+++ b/x-pack/test/fleet_api_integration/apis/epm/get.ts
@@ -6,6 +6,7 @@
  */
 
 import expect from '@kbn/expect';
+import { PackageInfo } from '@kbn/fleet-plugin/common/types/models/epm';
 import fs from 'fs';
 import path from 'path';
 import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
@@ -172,5 +173,42 @@ export default function (providerContext: FtrProviderContext) {
           .expect(200);
       });
     });
+    it('returns package info from the archive if ?full=true', async function () {
+      const res = await supertest
+        .get(`/api/fleet/epm/packages/non_epr_fields/0.2.0?full=true`)
+        .expect(200);
+      const packageInfo = res.body.item as PackageInfo;
+      expect(packageInfo?.data_streams?.length).equal(3);
+      const dataStream = packageInfo?.data_streams?.find(
+        ({ dataset }) => dataset === 'non_epr_fields.test_metrics_2'
+      );
+      expect(dataStream?.elasticsearch?.source_mode).equal('default');
+    });
+    it('returns package info from the registry if ?full=false', async function () {
+      const res = await supertest
+        .get(`/api/fleet/epm/packages/non_epr_fields/0.2.0?full=false`)
+        .expect(200);
+      const packageInfo = res.body.item as PackageInfo;
+      expect(packageInfo?.data_streams?.length).equal(3);
+      const dataStream = packageInfo?.data_streams?.find(
+        ({ dataset }) => dataset === 'non_epr_fields.test_metrics_2'
+      );
+      // this field is only returned if we go to the archive
+      // it i not part of the EPR API
+      expect(dataStream?.elasticsearch?.source_mode).equal(undefined);
+    });
+    it('returns package info from the registry if ?full not provided', async function () {
+      const res = await supertest
+        .get(`/api/fleet/epm/packages/non_epr_fields/0.2.0?full=false`)
+        .expect(200);
+      const packageInfo = res.body.item as PackageInfo;
+      expect(packageInfo?.data_streams?.length).equal(3);
+      const dataStream = packageInfo?.data_streams?.find(
+        ({ dataset }) => dataset === 'non_epr_fields.test_metrics_2'
+      );
+      // this field is only returned if we go to the archive
+      // it i not part of the EPR API
+      expect(dataStream?.elasticsearch?.source_mode).equal(undefined);
+    });
   });
 }

From 6b87dd81894fb6910eb0a722fa55d0f1c7d22ed9 Mon Sep 17 00:00:00 2001
From: Mark Hopkin <mark.hopkin@elastic.co>
Date: Tue, 1 Nov 2022 17:22:13 +0000
Subject: [PATCH 07/13] add new param to frontent

---
 .../create_package_policy_page/multi_page_layout/index.tsx    | 2 +-
 .../create_package_policy_page/single_page_layout/index.tsx   | 2 +-
 .../sections/agent_policy/edit_package_policy_page/index.tsx  | 2 +-
 x-pack/plugins/fleet/public/hooks/use_request/epm.ts          | 4 +++-
 4 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/multi_page_layout/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/multi_page_layout/index.tsx
index 46f12816f02c0..60b12f76eb8be 100644
--- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/multi_page_layout/index.tsx
+++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/multi_page_layout/index.tsx
@@ -70,7 +70,7 @@ export const CreatePackagePolicyMultiPage: CreatePackagePolicyParams = ({
     data: packageInfoData,
     error: packageInfoError,
     isLoading: isPackageInfoLoading,
-  } = useGetPackageInfoByKey(pkgName, pkgVersion, { prerelease: true });
+  } = useGetPackageInfoByKey(pkgName, pkgVersion, { prerelease: true, full: true });
 
   const {
     agentPolicy,
diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/index.tsx
index 0192b7a4a7928..ae2c067a95aee 100644
--- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/index.tsx
+++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/index.tsx
@@ -99,7 +99,7 @@ export const CreatePackagePolicySinglePage: CreatePackagePolicyParams = ({
     data: packageInfoData,
     error: packageInfoError,
     isLoading: isPackageInfoLoading,
-  } = useGetPackageInfoByKey(pkgName, pkgVersion, { prerelease: true });
+  } = useGetPackageInfoByKey(pkgName, pkgVersion, { full: true, prerelease: true });
   const packageInfo = useMemo(() => {
     if (packageInfoData && packageInfoData.item) {
       return packageInfoData.item;
diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/index.tsx
index aa9ca6277a43c..3802d7afa039c 100644
--- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/index.tsx
+++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/index.tsx
@@ -266,7 +266,7 @@ export const EditPackagePolicyForm = memo<{
             const { data: packageData } = await sendGetPackageInfoByKey(
               _packageInfo!.name,
               _packageInfo!.version,
-              { prerelease: true }
+              { prerelease: true, full: true }
             );
 
             if (packageData?.item) {
diff --git a/x-pack/plugins/fleet/public/hooks/use_request/epm.ts b/x-pack/plugins/fleet/public/hooks/use_request/epm.ts
index 9c88cfae46c4d..f1a248c44c07b 100644
--- a/x-pack/plugins/fleet/public/hooks/use_request/epm.ts
+++ b/x-pack/plugins/fleet/public/hooks/use_request/epm.ts
@@ -85,6 +85,7 @@ export const useGetPackageInfoByKey = (
   options?: {
     ignoreUnverified?: boolean;
     prerelease?: boolean;
+    full?: boolean;
   }
 ) => {
   const confirmOpenUnverified = useConfirmOpenUnverified();
@@ -96,7 +97,7 @@ export const useGetPackageInfoByKey = (
     method: 'get',
     query: {
       ...options,
-      ...(ignoreUnverifiedQueryParam ? { ignoreUnverified: ignoreUnverifiedQueryParam } : {}),
+      ...(ignoreUnverifiedQueryParam && { ignoreUnverified: ignoreUnverifiedQueryParam }),
     },
   });
 
@@ -130,6 +131,7 @@ export const sendGetPackageInfoByKey = (
   options?: {
     ignoreUnverified?: boolean;
     prerelease?: boolean;
+    full?: boolean;
   }
 ) => {
   return sendRequest<GetInfoResponse>({

From 1afaa7fda88bce849dc9f7fb7cde43f1587e6af3 Mon Sep 17 00:00:00 2001
From: Mark Hopkin <mark.hopkin@elastic.co>
Date: Thu, 3 Nov 2022 21:58:59 +0000
Subject: [PATCH 08/13] use released package for test

---
 x-pack/test/fleet_api_integration/apis/epm/get.ts           | 6 +++---
 .../{0.2.0 => 1.0.0}/data_stream/test_logs/fields/ecs.yml   | 0
 .../data_stream/test_logs/fields/fields.yml                 | 0
 .../{0.2.0 => 1.0.0}/data_stream/test_logs/manifest.yml     | 0
 .../data_stream/test_metrics/fields/ecs.yml                 | 0
 .../data_stream/test_metrics/fields/fields.yml              | 0
 .../{0.2.0 => 1.0.0}/data_stream/test_metrics/manifest.yml  | 0
 .../data_stream/test_metrics_2/fields/ecs.yml               | 0
 .../data_stream/test_metrics_2/fields/fields.yml            | 0
 .../data_stream/test_metrics_2/manifest.yml                 | 0
 .../non_epr_fields/{0.2.0 => 1.0.0}/docs/README.md          | 0
 .../non_epr_fields/{0.2.0 => 1.0.0}/manifest.yml            | 2 +-
 12 files changed, 4 insertions(+), 4 deletions(-)
 rename x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/{0.2.0 => 1.0.0}/data_stream/test_logs/fields/ecs.yml (100%)
 rename x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/{0.2.0 => 1.0.0}/data_stream/test_logs/fields/fields.yml (100%)
 rename x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/{0.2.0 => 1.0.0}/data_stream/test_logs/manifest.yml (100%)
 rename x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/{0.2.0 => 1.0.0}/data_stream/test_metrics/fields/ecs.yml (100%)
 rename x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/{0.2.0 => 1.0.0}/data_stream/test_metrics/fields/fields.yml (100%)
 rename x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/{0.2.0 => 1.0.0}/data_stream/test_metrics/manifest.yml (100%)
 rename x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/{0.2.0 => 1.0.0}/data_stream/test_metrics_2/fields/ecs.yml (100%)
 rename x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/{0.2.0 => 1.0.0}/data_stream/test_metrics_2/fields/fields.yml (100%)
 rename x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/{0.2.0 => 1.0.0}/data_stream/test_metrics_2/manifest.yml (100%)
 rename x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/{0.2.0 => 1.0.0}/docs/README.md (100%)
 rename x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/{0.2.0 => 1.0.0}/manifest.yml (98%)

diff --git a/x-pack/test/fleet_api_integration/apis/epm/get.ts b/x-pack/test/fleet_api_integration/apis/epm/get.ts
index 4834559ec2eac..845c27448d761 100644
--- a/x-pack/test/fleet_api_integration/apis/epm/get.ts
+++ b/x-pack/test/fleet_api_integration/apis/epm/get.ts
@@ -175,7 +175,7 @@ export default function (providerContext: FtrProviderContext) {
     });
     it('returns package info from the archive if ?full=true', async function () {
       const res = await supertest
-        .get(`/api/fleet/epm/packages/non_epr_fields/0.2.0?full=true`)
+        .get(`/api/fleet/epm/packages/non_epr_fields/1.0.0?full=true`)
         .expect(200);
       const packageInfo = res.body.item as PackageInfo;
       expect(packageInfo?.data_streams?.length).equal(3);
@@ -186,7 +186,7 @@ export default function (providerContext: FtrProviderContext) {
     });
     it('returns package info from the registry if ?full=false', async function () {
       const res = await supertest
-        .get(`/api/fleet/epm/packages/non_epr_fields/0.2.0?full=false`)
+        .get(`/api/fleet/epm/packages/non_epr_fields/1.0.0?full=false`)
         .expect(200);
       const packageInfo = res.body.item as PackageInfo;
       expect(packageInfo?.data_streams?.length).equal(3);
@@ -199,7 +199,7 @@ export default function (providerContext: FtrProviderContext) {
     });
     it('returns package info from the registry if ?full not provided', async function () {
       const res = await supertest
-        .get(`/api/fleet/epm/packages/non_epr_fields/0.2.0?full=false`)
+        .get(`/api/fleet/epm/packages/non_epr_fields/1.0.0?full=false`)
         .expect(200);
       const packageInfo = res.body.item as PackageInfo;
       expect(packageInfo?.data_streams?.length).equal(3);
diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_logs/fields/ecs.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/1.0.0/data_stream/test_logs/fields/ecs.yml
similarity index 100%
rename from x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_logs/fields/ecs.yml
rename to x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/1.0.0/data_stream/test_logs/fields/ecs.yml
diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_logs/fields/fields.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/1.0.0/data_stream/test_logs/fields/fields.yml
similarity index 100%
rename from x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_logs/fields/fields.yml
rename to x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/1.0.0/data_stream/test_logs/fields/fields.yml
diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_logs/manifest.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/1.0.0/data_stream/test_logs/manifest.yml
similarity index 100%
rename from x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_logs/manifest.yml
rename to x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/1.0.0/data_stream/test_logs/manifest.yml
diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics/fields/ecs.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/1.0.0/data_stream/test_metrics/fields/ecs.yml
similarity index 100%
rename from x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics/fields/ecs.yml
rename to x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/1.0.0/data_stream/test_metrics/fields/ecs.yml
diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics/fields/fields.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/1.0.0/data_stream/test_metrics/fields/fields.yml
similarity index 100%
rename from x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics/fields/fields.yml
rename to x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/1.0.0/data_stream/test_metrics/fields/fields.yml
diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics/manifest.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/1.0.0/data_stream/test_metrics/manifest.yml
similarity index 100%
rename from x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics/manifest.yml
rename to x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/1.0.0/data_stream/test_metrics/manifest.yml
diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics_2/fields/ecs.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/1.0.0/data_stream/test_metrics_2/fields/ecs.yml
similarity index 100%
rename from x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics_2/fields/ecs.yml
rename to x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/1.0.0/data_stream/test_metrics_2/fields/ecs.yml
diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics_2/fields/fields.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/1.0.0/data_stream/test_metrics_2/fields/fields.yml
similarity index 100%
rename from x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics_2/fields/fields.yml
rename to x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/1.0.0/data_stream/test_metrics_2/fields/fields.yml
diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics_2/manifest.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/1.0.0/data_stream/test_metrics_2/manifest.yml
similarity index 100%
rename from x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/data_stream/test_metrics_2/manifest.yml
rename to x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/1.0.0/data_stream/test_metrics_2/manifest.yml
diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/docs/README.md b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/1.0.0/docs/README.md
similarity index 100%
rename from x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/docs/README.md
rename to x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/1.0.0/docs/README.md
diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/manifest.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/1.0.0/manifest.yml
similarity index 98%
rename from x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/manifest.yml
rename to x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/1.0.0/manifest.yml
index 29b5dc60149c8..9892190206acf 100644
--- a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/0.2.0/manifest.yml
+++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/non_epr_fields/1.0.0/manifest.yml
@@ -2,7 +2,7 @@ format_version: 1.0.0
 name: non_epr_fields
 title: Non EPR Fields
 description: This package contain fields in its manifest and datastreams such as source_mode which are not returned by the EPR API.
-version: 0.2.0
+version: 1.0.0
 categories: []
 release: beta
 type: integration

From 37a416c537596dc63524338c15971e9ed3f7615f Mon Sep 17 00:00:00 2001
From: Mark Hopkin <mark.hopkin@elastic.co>
Date: Fri, 4 Nov 2022 16:32:46 +0000
Subject: [PATCH 09/13] tidy for PR

---
 x-pack/plugins/fleet/common/types/models/epm.ts           | 4 ----
 x-pack/plugins/fleet/server/services/epm/archive/cache.ts | 5 +++++
 x-pack/test/fleet_api_integration/apis/epm/get.ts         | 4 +---
 3 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts
index 8341dcef581f1..4ef4a7fdd751b 100644
--- a/x-pack/plugins/fleet/common/types/models/epm.ts
+++ b/x-pack/plugins/fleet/common/types/models/epm.ts
@@ -354,10 +354,6 @@ export interface RegistryElasticsearch {
   source_mode?: 'default' | 'synthetic';
 }
 
-export type ArchiveElasticsearch = RegistryElasticsearch & {
-  source_mode?: 'default' | 'synthetic';
-};
-
 export interface RegistryDataStreamPrivileges {
   cluster?: string[];
   indices?: string[];
diff --git a/x-pack/plugins/fleet/server/services/epm/archive/cache.ts b/x-pack/plugins/fleet/server/services/epm/archive/cache.ts
index 7b9aae17e68ab..8b1fd141f3000 100644
--- a/x-pack/plugins/fleet/server/services/epm/archive/cache.ts
+++ b/x-pack/plugins/fleet/server/services/epm/archive/cache.ts
@@ -67,6 +67,11 @@ export const getArchivePackage = (args: SharedKey) => {
   };
 };
 
+/*
+ * This cache should only be used to store "full" package info generated from the package archive.
+ * NOT package info from the EPR API. This is because we parse extra fields from the archive
+ * which are not provided by the registry API.
+ */
 export const setPackageInfo = ({
   name,
   version,
diff --git a/x-pack/test/fleet_api_integration/apis/epm/get.ts b/x-pack/test/fleet_api_integration/apis/epm/get.ts
index 845c27448d761..42e78b731a104 100644
--- a/x-pack/test/fleet_api_integration/apis/epm/get.ts
+++ b/x-pack/test/fleet_api_integration/apis/epm/get.ts
@@ -194,7 +194,7 @@ export default function (providerContext: FtrProviderContext) {
         ({ dataset }) => dataset === 'non_epr_fields.test_metrics_2'
       );
       // this field is only returned if we go to the archive
-      // it i not part of the EPR API
+      // it is not part of the EPR API
       expect(dataStream?.elasticsearch?.source_mode).equal(undefined);
     });
     it('returns package info from the registry if ?full not provided', async function () {
@@ -206,8 +206,6 @@ export default function (providerContext: FtrProviderContext) {
       const dataStream = packageInfo?.data_streams?.find(
         ({ dataset }) => dataset === 'non_epr_fields.test_metrics_2'
       );
-      // this field is only returned if we go to the archive
-      // it i not part of the EPR API
       expect(dataStream?.elasticsearch?.source_mode).equal(undefined);
     });
   });

From 60ee46aa5d626e1d7504c60ed04d91413deaea6e Mon Sep 17 00:00:00 2001
From: Mark Hopkin <mark.hopkin@elastic.co>
Date: Fri, 4 Nov 2022 16:55:51 +0000
Subject: [PATCH 10/13] Add API docs

---
 x-pack/plugins/fleet/common/openapi/bundled.json          | 8 ++++++++
 x-pack/plugins/fleet/common/openapi/bundled.yaml          | 7 +++++++
 .../paths/epm@packages@{pkg_name}@{pkg_version}.yaml      | 5 +++++
 3 files changed, 20 insertions(+)

diff --git a/x-pack/plugins/fleet/common/openapi/bundled.json b/x-pack/plugins/fleet/common/openapi/bundled.json
index 42db2ff330680..de00930837ec1 100644
--- a/x-pack/plugins/fleet/common/openapi/bundled.json
+++ b/x-pack/plugins/fleet/common/openapi/bundled.json
@@ -667,6 +667,14 @@
           "name": "ignoreUnverified",
           "description": "Ignore if the package is fails signature verification",
           "in": "query"
+        },
+        {
+          "schema": {
+            "type": "boolean"
+          },
+          "name": "full",
+          "description": "Return all fields from the package manifest, not just those supported by the Elastic Package Registry",
+          "in": "query"
         }
       ],
       "post": {
diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml
index 08581dea73259..f3a94715d66c2 100644
--- a/x-pack/plugins/fleet/common/openapi/bundled.yaml
+++ b/x-pack/plugins/fleet/common/openapi/bundled.yaml
@@ -419,6 +419,13 @@ paths:
         name: ignoreUnverified
         description: Ignore if the package is fails signature verification
         in: query
+      - schema:
+          type: boolean
+        name: full
+        description: >-
+          Return all fields from the package manifest, not just those supported
+          by the Elastic Package Registry
+        in: query
     post:
       summary: Packages - Install
       tags: []
diff --git a/x-pack/plugins/fleet/common/openapi/paths/epm@packages@{pkg_name}@{pkg_version}.yaml b/x-pack/plugins/fleet/common/openapi/paths/epm@packages@{pkg_name}@{pkg_version}.yaml
index b0ef55cb7e52d..3e3cbbb0c1f56 100644
--- a/x-pack/plugins/fleet/common/openapi/paths/epm@packages@{pkg_name}@{pkg_version}.yaml
+++ b/x-pack/plugins/fleet/common/openapi/paths/epm@packages@{pkg_name}@{pkg_version}.yaml
@@ -52,6 +52,11 @@ parameters:
     name: ignoreUnverified
     description: 'Ignore if the package is fails signature verification'
     in: query
+  - schema:
+      type: boolean
+    name: full
+    description: 'Return all fields from the package manifest, not just those supported by the Elastic Package Registry'
+    in: query
 post:
   summary: Packages - Install
   tags: []

From 5100f06d746efa0d8a2dc34c978973cc18f5dde2 Mon Sep 17 00:00:00 2001
From: Mark Hopkin <mark.hopkin@elastic.co>
Date: Mon, 7 Nov 2022 15:36:41 +0000
Subject: [PATCH 11/13] add get all packages script

---
 .../get_all_packages/get_all_packages.ts      | 128 ++++++++++++++++++
 .../fleet/scripts/get_all_packages/index.js   |   9 ++
 2 files changed, 137 insertions(+)
 create mode 100644 x-pack/plugins/fleet/scripts/get_all_packages/get_all_packages.ts
 create mode 100644 x-pack/plugins/fleet/scripts/get_all_packages/index.js

diff --git a/x-pack/plugins/fleet/scripts/get_all_packages/get_all_packages.ts b/x-pack/plugins/fleet/scripts/get_all_packages/get_all_packages.ts
new file mode 100644
index 0000000000000..9effa021343f3
--- /dev/null
+++ b/x-pack/plugins/fleet/scripts/get_all_packages/get_all_packages.ts
@@ -0,0 +1,128 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import fetch from 'node-fetch';
+import { kibanaPackageJson } from '@kbn/utils';
+import { ToolingLog } from '@kbn/tooling-log';
+import { chunk } from 'lodash';
+
+import yargs from 'yargs/yargs';
+
+import type { PackageInfo } from '../../common';
+
+const REGISTRY_URL = 'https://epr-snapshot.elastic.co';
+const KIBANA_URL = 'http://localhost:5601';
+const KIBANA_USERNAME = 'elastic';
+const KIBANA_PASSWORD = 'changeme';
+const KIBANA_VERSION = kibanaPackageJson.version;
+
+const { base = '', help, prerelease = false, batchSize = 1 } = yargs(process.argv).argv;
+
+const logger = new ToolingLog({
+  level: 'info',
+  writeTo: process.stdout,
+});
+
+interface Result {
+  pkg: string;
+  epr: number;
+  archive: number;
+  archiveCached: number;
+}
+async function getPackage(name: string, version: string, full: boolean = false) {
+  const start = Date.now();
+  const res = await fetch(
+    `${KIBANA_URL}${base}/api/fleet/epm/packages/${name}/${version}?prerelease=true${
+      full ? '&full=true' : ''
+    }`,
+    {
+      headers: {
+        accept: '*/*',
+        'content-type': 'application/json',
+        'kbn-xsrf': 'xyz',
+        Authorization:
+          'Basic ' + Buffer.from(`${KIBANA_USERNAME}:${KIBANA_PASSWORD}`).toString('base64'),
+      },
+      method: 'GET',
+    }
+  );
+  const end = Date.now();
+
+  let body;
+  try {
+    body = await res.json();
+  } catch (e) {
+    logger.error(`Error parsing response: ${e}`);
+    throw e;
+  }
+
+  if (body.item && body.item.name) {
+    return { pkg: body.item, status: body.status, took: (end - start) / 1000 };
+  }
+
+  throw new Error(`Invalid package returned for ${name}-${version} : ${JSON.stringify(res)}`);
+}
+
+async function getAllPackages() {
+  const res = await fetch(
+    `${REGISTRY_URL}/search?kibana.version=${KIBANA_VERSION}${
+      prerelease ? '&prerelease=true' : ''
+    }`,
+    {
+      headers: {
+        accept: '*/*',
+      },
+      method: 'GET',
+    }
+  );
+  const body = await res.json();
+  return body as PackageInfo[];
+}
+
+async function performTest({ name, version }: { name: string; version: string }): Promise<Result> {
+  const eprResult = await getPackage(name, version);
+  const archiveResult = await getPackage(name, version, true);
+  const cachedArchiveResult = await getPackage(name, version, true);
+  logger.info(`✅ ${name}-${version}`);
+
+  return {
+    pkg: `${name}-${version}`,
+    epr: eprResult.took,
+    archive: archiveResult.took,
+    archiveCached: cachedArchiveResult.took,
+  };
+}
+
+export async function run() {
+  const allPackages = await getAllPackages();
+
+  const batches = chunk(allPackages, batchSize);
+  let allResults: Result[] = [];
+
+  const start = Date.now();
+  for (const batch of batches) {
+    const results = await Promise.all(batch.map(performTest));
+    allResults = [...allResults, ...(results.filter((v) => v) as Result[])];
+  }
+  const end = Date.now();
+  const took = (end - start) / 1000;
+  allResults.sort((a, b) => b.archive - a.archive);
+  logger.info(`Took ${took} seconds to get ${allResults.length} packages`);
+  logger.info(
+    'Average EPM time: ' + allResults.reduce((acc, { epr }) => acc + epr, 0) / allResults.length
+  );
+  logger.info(
+    'Average Archive time: ' +
+      allResults.reduce((acc, { archive }) => acc + archive, 0) / allResults.length
+  );
+  logger.info(
+    'Average Cache time: ' +
+      allResults.reduce((acc, { archiveCached }) => acc + archiveCached, 0) / allResults.length
+  );
+  // eslint-disable-next-line no-console
+  console.table(allResults);
+}
diff --git a/x-pack/plugins/fleet/scripts/get_all_packages/index.js b/x-pack/plugins/fleet/scripts/get_all_packages/index.js
new file mode 100644
index 0000000000000..447b09c6b1b3f
--- /dev/null
+++ b/x-pack/plugins/fleet/scripts/get_all_packages/index.js
@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+require('../../../../../src/setup_node_env');
+require('./get_all_packages').run();

From 722346bb83b3d55dccffafb45c2decb1e1bad73c Mon Sep 17 00:00:00 2001
From: Mark Hopkin <mark.hopkin@elastic.co>
Date: Mon, 7 Nov 2022 16:33:00 +0000
Subject: [PATCH 12/13] get archive from cache even if not installed

---
 .../plugins/fleet/server/services/epm/packages/get.ts  | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts
index e8d1cd1380303..3c4861b563b08 100644
--- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts
+++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts
@@ -300,8 +300,14 @@ export async function getPackageFromSource(options: {
       }
     }
   } else {
-    res = await Registry.getPackage(pkgName, pkgVersion, { ignoreUnverified });
-    logger.debug(`retrieved package ${pkgName}-${pkgVersion} from registry`);
+    res = getArchivePackage({ name: pkgName, version: pkgVersion });
+
+    if (res) {
+      logger.debug(`retrieved package ${pkgName}-${pkgVersion} from cache`);
+    } else {
+      res = await Registry.getPackage(pkgName, pkgVersion, { ignoreUnverified });
+      logger.debug(`retrieved package ${pkgName}-${pkgVersion} from registry`);
+    }
   }
   if (!res) {
     throw new FleetError(`package info for ${pkgName}-${pkgVersion} does not exist`);

From e9a61367bc6e6f24284d4c80433947963b541582 Mon Sep 17 00:00:00 2001
From: Mark Hopkin <mark.hopkin@elastic.co>
Date: Mon, 7 Nov 2022 16:33:52 +0000
Subject: [PATCH 13/13] fix types

---
 .../fleet/scripts/get_all_packages/get_all_packages.ts        | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/x-pack/plugins/fleet/scripts/get_all_packages/get_all_packages.ts b/x-pack/plugins/fleet/scripts/get_all_packages/get_all_packages.ts
index 9effa021343f3..74ea305b45bb9 100644
--- a/x-pack/plugins/fleet/scripts/get_all_packages/get_all_packages.ts
+++ b/x-pack/plugins/fleet/scripts/get_all_packages/get_all_packages.ts
@@ -20,7 +20,7 @@ const KIBANA_USERNAME = 'elastic';
 const KIBANA_PASSWORD = 'changeme';
 const KIBANA_VERSION = kibanaPackageJson.version;
 
-const { base = '', help, prerelease = false, batchSize = 1 } = yargs(process.argv).argv;
+const { base = '', prerelease = false, batchSize = 1 } = yargs(process.argv).argv;
 
 const logger = new ToolingLog({
   level: 'info',
@@ -100,7 +100,7 @@ async function performTest({ name, version }: { name: string; version: string })
 export async function run() {
   const allPackages = await getAllPackages();
 
-  const batches = chunk(allPackages, batchSize);
+  const batches = chunk(allPackages, batchSize as number);
   let allResults: Result[] = [];
 
   const start = Date.now();