[Fleet] Add traces to Fleet setup on Kibana boot (#171836)

## Summary

Several improvements to tracing and logging in Fleet setup to find root
cause of problems:
- Add a trace transaction to ensure Fleet setup calls on Kibana start up
are traced
- Add specific spans around each step of setup preconfiguration to more
easily correlate errors to the process
- Capture errors in APM errors
- Add stack traces to error logs during package installation to better
identify the cause of the error

---------

Co-authored-by: kibanamachine <[email protected]>

x-pack/plugins/fleet/server/services/epm/packages/install.ts

@@ -597,7 +597,9 @@ async function installPackageCommon(options: {
         return { assets, status: 'installed', installType, installSource };
       })
       .catch(async (err: Error) => {
-        logger.warn(`Failure to install package [${pkgName}]: [${err.toString()}]`);
+        logger.warn(`Failure to install package [${pkgName}]: [${err.toString()}]`, {
+          error: { stack_trace: err.stack },
+        });
         await handleInstallPackageFailure({
           savedObjectsClient,
           error: err,

x-pack/plugins/fleet/server/services/preconfiguration.ts

@@ -255,10 +255,7 @@ export async function ensurePreconfiguredPackagesAndPolicies(
         );
       });
 
-      apm.startTransaction(
-        'fleet.preconfiguration.addPackagePolicies.improved.prReview.50',
-        'fleet'
-      );
+      const s = apm.startSpan('Add preconfigured package policies', 'preconfiguration');
       await addPreconfiguredPolicyPackages(
         soClient,
         esClient,
@@ -267,7 +264,7 @@ export async function ensurePreconfiguredPackagesAndPolicies(
         defaultOutput,
         true
       );
-      apm.endTransaction('fleet.preconfiguration.addPackagePolicies.improved.prReview.50');
+      s?.end();
 
       // Add the is_managed flag after configuring package policies to avoid errors
       if (shouldAddIsManagedFlag) {

x-pack/plugins/fleet/server/services/setup.ts

@@ -6,6 +6,7 @@
  */
 
 import fs from 'fs/promises';
+import apm from 'elastic-apm-node';
 
 import { compact } from 'lodash';
 import pMap from 'p-map';
@@ -61,7 +62,17 @@ export async function setupFleet(
   soClient: SavedObjectsClientContract,
   esClient: ElasticsearchClient
 ): Promise<SetupStatus> {
-  return awaitIfPending(async () => createSetupSideEffects(soClient, esClient));
+  const t = apm.startTransaction('fleet-setup', 'fleet');
+
+  try {
+    return await awaitIfPending(async () => createSetupSideEffects(soClient, esClient));
+  } catch (error) {
+    apm.captureError(error);
+    t.setOutcome('failure');
+    throw error;
+  } finally {
+    t.end();
+  }
 }
 
 async function createSetupSideEffects(
@@ -113,7 +124,9 @@ async function createSetupSideEffects(
   const defaultOutput = await outputService.ensureDefaultOutput(soClient, esClient);
 
   logger.debug('Setting up Fleet Elasticsearch assets');
+  let stepSpan = apm.startSpan('Install Fleet global assets', 'preconfiguration');
   await ensureFleetGlobalEsAssets(soClient, esClient);
+  stepSpan?.end();
 
   // Ensure that required packages are always installed even if they're left out of the config
   const preconfiguredPackageNames = new Set(packages.map((pkg) => pkg.name));
@@ -136,6 +149,7 @@ async function createSetupSideEffects(
 
   logger.debug('Setting up initial Fleet packages');
 
+  stepSpan = apm.startSpan('Install preconfigured packages and policies', 'preconfiguration');
   const { nonFatalErrors: preconfiguredPackagesNonFatalErrors } =
     await ensurePreconfiguredPackagesAndPolicies(
       soClient,
@@ -146,17 +160,23 @@ async function createSetupSideEffects(
       defaultDownloadSource,
       DEFAULT_SPACE_ID
     );
+  stepSpan?.end();
 
+  stepSpan = apm.startSpan('Upgrade managed package policies', 'preconfiguration');
   const packagePolicyUpgradeErrors = (
     await upgradeManagedPackagePolicies(soClient, esClient)
   ).filter((result) => (result.errors ?? []).length > 0);
+  stepSpan?.end();
 
   const nonFatalErrors = [...preconfiguredPackagesNonFatalErrors, ...packagePolicyUpgradeErrors];
 
   logger.debug('Upgrade Fleet package install versions');
+  stepSpan = apm.startSpan('Upgrade package install format version', 'preconfiguration');
   await upgradePackageInstallVersion({ soClient, esClient, logger });
+  stepSpan?.end();
 
   logger.debug('Generating key pair for message signing');
+  stepSpan = apm.startSpan('Configure message signing', 'preconfiguration');
   if (!appContextService.getMessageSigningService()?.isEncryptionAvailable) {
     logger.warn(
       'xpack.encryptedSavedObjects.encryptionKey is not configured, private key passphrase is being stored in plain text'
@@ -176,16 +196,26 @@ async function createSetupSideEffects(
     logger.debug('Checking for and encrypting plain text uninstall tokens');
     await appContextService.getUninstallTokenService()?.encryptTokens();
   }
+  stepSpan?.end();
 
+  stepSpan = apm.startSpan('Upgrade agent policy schema', 'preconfiguration');
   logger.debug('Upgrade Agent policy schema version');
   await upgradeAgentPolicySchemaVersion(soClient);
+  stepSpan?.end();
 
+  stepSpan = apm.startSpan('Set up enrollment keys for preconfigured policies', 'preconfiguration');
   logger.debug('Setting up Fleet enrollment keys');
   await ensureDefaultEnrollmentAPIKeysExists(soClient, esClient);
+  stepSpan?.end();
 
   if (nonFatalErrors.length > 0) {
     logger.info('Encountered non fatal errors during Fleet setup');
-    formatNonFatalErrors(nonFatalErrors).forEach((error) => logger.info(JSON.stringify(error)));
+    formatNonFatalErrors(nonFatalErrors)
+      .map((e) => JSON.stringify(e))
+      .forEach((error) => {
+        logger.info(error);
+        apm.captureError(error);
+      });
   }
 
   logger.info('Fleet setup completed');
@@ -222,6 +252,7 @@ export async function ensureFleetGlobalEsAssets(
           esClient,
           installation,
         }).catch((err) => {
+          apm.captureError(err);
           logger.error(
             `Package needs to be manually reinstalled ${installation.name} after installing Fleet global assets: ${err.message}`
           );

x-pack/plugins/fleet/server/services/setup_utils.ts

@@ -12,7 +12,7 @@ let isPending = false;
 let onResolve = (value?: unknown) => {};
 let onReject = (reason: any) => {};
 
-export async function awaitIfPending<T>(asyncFunction: Function): Promise<T> {
+export async function awaitIfPending<T>(asyncFunction: () => Promise<T>): Promise<T> {
   // pending successful or failed attempt
   if (isPending) {
     // don't run concurrent installs