From 1e13b0d2c4c1f5aa1700111ac4f8d64f8870a848 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Wed, 26 Oct 2022 09:21:10 -0400 Subject: [PATCH 1/3] [Fleet] Log agent usage every n minutes --- .../scripts/create_agents/create_agents.ts | 2 - .../fleet/server/collectors/register.ts | 10 +++ x-pack/plugins/fleet/server/plugin.ts | 5 +- .../server/services/fleet_usage_logger.ts | 63 +++++++++++++++++++ 4 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugins/fleet/server/services/fleet_usage_logger.ts diff --git a/x-pack/plugins/fleet/scripts/create_agents/create_agents.ts b/x-pack/plugins/fleet/scripts/create_agents/create_agents.ts index 31df24c70a5d4..0a9d14339729a 100644 --- a/x-pack/plugins/fleet/scripts/create_agents/create_agents.ts +++ b/x-pack/plugins/fleet/scripts/create_agents/create_agents.ts @@ -99,8 +99,6 @@ async function createAgentPolicy(id: string) { namespace: 'default', description: '', monitoring_enabled: ['logs'], - data_output_id: 'fleet-default-output', - monitoring_output_id: 'fleet-default-output', }), headers: { Authorization: auth, diff --git a/x-pack/plugins/fleet/server/collectors/register.ts b/x-pack/plugins/fleet/server/collectors/register.ts index f19682a5a243a..a194ff9b560e5 100644 --- a/x-pack/plugins/fleet/server/collectors/register.ts +++ b/x-pack/plugins/fleet/server/collectors/register.ts @@ -37,6 +37,16 @@ export const fetchUsage = async (core: CoreSetup, config: FleetConfigType) => { return usage; }; +export const fetchAgentsUsage = async (core: CoreSetup, config: FleetConfigType) => { + const [soClient, esClient] = await getInternalClients(core); + const usage = { + agents_enabled: getIsAgentsEnabled(config), + agents: await getAgentUsage(config, soClient, esClient), + fleet_server: await getFleetServerUsage(soClient, esClient), + }; + return usage; +}; + export function registerFleetUsageCollector( core: CoreSetup, config: FleetConfigType, diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index 7d00f944fbf81..5d177641daee5 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -101,7 +101,7 @@ import { AgentServiceImpl, PackageServiceImpl, } from './services'; -import { registerFleetUsageCollector, fetchUsage } from './collectors/register'; +import { registerFleetUsageCollector, fetchUsage, fetchAgentsUsage } from './collectors/register'; import { getAuthzFromRequest, makeRouterWithFleetAuthz } from './routes/security'; import { FleetArtifactsClient } from './services/artifacts'; import type { FleetRouter } from './types/request_context'; @@ -110,6 +110,7 @@ import { setupFleet } from './services/setup'; import { BulkActionsResolver } from './services/agents'; import type { PackagePolicyService } from './services/package_policy_service'; import { PackagePolicyServiceImpl } from './services/package_policy'; +import { registerFleetUsageLogger, startFleetUsageLogger } from './services/fleet_usage_logger'; export interface FleetSetupDeps { security: SecurityPluginSetup; @@ -388,6 +389,7 @@ export class FleetPlugin this.kibanaVersion, this.isProductionMode ); + registerFleetUsageLogger(deps.taskManager, async () => fetchAgentsUsage(core, config)); const router: FleetRouter = core.http.createRouter(); // Allow read-only users access to endpoints necessary for Integrations UI @@ -455,6 +457,7 @@ export class FleetPlugin this.telemetryEventsSender.start(plugins.telemetry, core); this.bulkActionsResolver?.start(plugins.taskManager); this.fleetUsageSender?.start(plugins.taskManager); + startFleetUsageLogger(plugins.taskManager); const logger = appContextService.getLogger(); diff --git a/x-pack/plugins/fleet/server/services/fleet_usage_logger.ts b/x-pack/plugins/fleet/server/services/fleet_usage_logger.ts new file mode 100644 index 0000000000000..c9af363ee0118 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/fleet_usage_logger.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { + ConcreteTaskInstance, + TaskManagerStartContract, + TaskManagerSetupContract, +} from '@kbn/task-manager-plugin/server'; + +import type { fetchAgentsUsage } from '../collectors/register'; + +import { appContextService } from './app_context'; + +const TASK_ID = 'Fleet-Usage-Logger-Task'; +const TASK_TYPE = 'Fleet-Usage-Logger'; + +export async function registerFleetUsageLogger( + taskManager: TaskManagerSetupContract, + fetchUsage: () => ReturnType +) { + taskManager.registerTaskDefinitions({ + [TASK_TYPE]: { + title: 'Fleet Usage Logger', + timeout: '1m', + maxAttempts: 1, + createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { + return { + async run() { + try { + const usageData = await fetchUsage(); + // TODO get log level and use debug or info based on that + appContextService.getLogger().info(`Feet Usage: ${JSON.stringify(usageData)}`); + } catch (error) { + appContextService + .getLogger() + .error('Error occurred while fetching fleet usage: ' + error); + } + }, + + async cancel() {}, + }; + }, + }, + }); +} + +export async function startFleetUsageLogger(taskManager: TaskManagerStartContract) { + // TODO get log level and schedule interval based on that + appContextService.getLogger().info(`Task ${TASK_ID} scheduled with interval 5m`); + await taskManager?.ensureScheduled({ + id: TASK_ID, + taskType: TASK_TYPE, + schedule: { + interval: '5m', + }, + scope: ['fleet'], + state: {}, + params: {}, + }); +} From ea1ac6b4f4bc24be8a9fb181dffe9347f1e59fc4 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Mon, 31 Oct 2022 11:06:58 -0400 Subject: [PATCH 2/3] Update to use new logger abbility --- .../fleet/server/services/fleet_usage_logger.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/fleet_usage_logger.ts b/x-pack/plugins/fleet/server/services/fleet_usage_logger.ts index c9af363ee0118..be28f59693eb0 100644 --- a/x-pack/plugins/fleet/server/services/fleet_usage_logger.ts +++ b/x-pack/plugins/fleet/server/services/fleet_usage_logger.ts @@ -31,8 +31,11 @@ export async function registerFleetUsageLogger( async run() { try { const usageData = await fetchUsage(); - // TODO get log level and use debug or info based on that - appContextService.getLogger().info(`Feet Usage: ${JSON.stringify(usageData)}`); + if (appContextService.getLogger().isLevelEnabled('debug')) { + appContextService.getLogger().debug(`Feet Usage: ${JSON.stringify(usageData)}`); + } else { + appContextService.getLogger().info(`Feet Usage: ${JSON.stringify(usageData)}`); + } } catch (error) { appContextService .getLogger() @@ -48,13 +51,18 @@ export async function registerFleetUsageLogger( } export async function startFleetUsageLogger(taskManager: TaskManagerStartContract) { + const isDebugLogLevelEnabled = appContextService.getLogger().isLevelEnabled('debug'); + const isInfoLogLevelEnabled = appContextService.getLogger().isLevelEnabled('info'); + if (!isInfoLogLevelEnabled) { + return; + } // TODO get log level and schedule interval based on that appContextService.getLogger().info(`Task ${TASK_ID} scheduled with interval 5m`); await taskManager?.ensureScheduled({ id: TASK_ID, taskType: TASK_TYPE, schedule: { - interval: '5m', + interval: isDebugLogLevelEnabled ? '5m' : '15m', }, scope: ['fleet'], state: {}, From baa38cdfb20ddc9bb0f2fa22543f3562965509cc Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Mon, 31 Oct 2022 11:22:35 -0400 Subject: [PATCH 3/3] Fix after review --- x-pack/plugins/fleet/server/services/fleet_usage_logger.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/fleet_usage_logger.ts b/x-pack/plugins/fleet/server/services/fleet_usage_logger.ts index be28f59693eb0..6aa84262dc541 100644 --- a/x-pack/plugins/fleet/server/services/fleet_usage_logger.ts +++ b/x-pack/plugins/fleet/server/services/fleet_usage_logger.ts @@ -32,9 +32,9 @@ export async function registerFleetUsageLogger( try { const usageData = await fetchUsage(); if (appContextService.getLogger().isLevelEnabled('debug')) { - appContextService.getLogger().debug(`Feet Usage: ${JSON.stringify(usageData)}`); + appContextService.getLogger().debug(`Fleet Usage: ${JSON.stringify(usageData)}`); } else { - appContextService.getLogger().info(`Feet Usage: ${JSON.stringify(usageData)}`); + appContextService.getLogger().info(`Fleet Usage: ${JSON.stringify(usageData)}`); } } catch (error) { appContextService @@ -56,7 +56,6 @@ export async function startFleetUsageLogger(taskManager: TaskManagerStartContrac if (!isInfoLogLevelEnabled) { return; } - // TODO get log level and schedule interval based on that appContextService.getLogger().info(`Task ${TASK_ID} scheduled with interval 5m`); await taskManager?.ensureScheduled({ id: TASK_ID,