Use DNS caching (elastic#184760)

Co-authored-by: Kibana Machine <[email protected]>
Co-authored-by: Jean-Louis Leysens <[email protected]>

package.json

@@ -980,6 +980,7 @@
     "brace": "0.11.1",
     "brok": "^5.0.2",
     "byte-size": "^8.1.0",
+    "cacheable-lookup": "6",
     "camelcase-keys": "7.0.2",
     "canvg": "^3.0.9",
     "cbor-x": "^1.3.3",

packages/core/elasticsearch/core-elasticsearch-client-server-internal/src/agent_manager.test.ts

@@ -33,7 +33,7 @@ describe('AgentManager', () => {
 
   describe('#getAgentFactory()', () => {
     it('provides factories which are different at each call', () => {
-      const agentManager = new AgentManager(logger);
+      const agentManager = new AgentManager(logger, { dnsCacheTtlInSeconds: 0 });
       const agentFactory1 = agentManager.getAgentFactory();
       const agentFactory2 = agentManager.getAgentFactory();
       expect(agentFactory1).not.toEqual(agentFactory2);
@@ -45,7 +45,7 @@ describe('AgentManager', () => {
         HttpAgentMock.mockImplementationOnce(() => mockedHttpAgent);
         const mockedHttpsAgent = new HttpsAgent();
         HttpsAgentMock.mockImplementationOnce(() => mockedHttpsAgent);
-        const agentManager = new AgentManager(logger);
+        const agentManager = new AgentManager(logger, { dnsCacheTtlInSeconds: 0 });
         const agentFactory = agentManager.getAgentFactory();
         const httpAgent = agentFactory({ url: new URL('http://elastic-node-1:9200') });
         const httpsAgent = agentFactory({ url: new URL('https://elastic-node-1:9200') });
@@ -54,7 +54,7 @@ describe('AgentManager', () => {
       });
 
       it('takes into account the provided configurations', () => {
-        const agentManager = new AgentManager(logger);
+        const agentManager = new AgentManager(logger, { dnsCacheTtlInSeconds: 0 });
         const agentFactory = agentManager.getAgentFactory({
           maxTotalSockets: 1024,
           scheduling: 'fifo',
@@ -77,7 +77,7 @@ describe('AgentManager', () => {
       });
 
       it('provides Agents that match the URLs protocol', () => {
-        const agentManager = new AgentManager(logger);
+        const agentManager = new AgentManager(logger, { dnsCacheTtlInSeconds: 0 });
         const agentFactory = agentManager.getAgentFactory();
         agentFactory({ url: new URL('http://elastic-node-1:9200') });
         expect(HttpAgent).toHaveBeenCalledTimes(1);
@@ -88,7 +88,7 @@ describe('AgentManager', () => {
       });
 
       it('provides the same Agent if URLs use the same protocol', () => {
-        const agentManager = new AgentManager(logger);
+        const agentManager = new AgentManager(logger, { dnsCacheTtlInSeconds: 0 });
         const agentFactory = agentManager.getAgentFactory();
         const agent1 = agentFactory({ url: new URL('http://elastic-node-1:9200') });
         const agent2 = agentFactory({ url: new URL('http://elastic-node-2:9200') });
@@ -101,7 +101,7 @@ describe('AgentManager', () => {
       });
 
       it('dereferences an agent instance when the agent is closed', () => {
-        const agentManager = new AgentManager(logger);
+        const agentManager = new AgentManager(logger, { dnsCacheTtlInSeconds: 0 });
         const agentFactory = agentManager.getAgentFactory();
         const agent = agentFactory({ url: new URL('http://elastic-node-1:9200') });
         // eslint-disable-next-line dot-notation
@@ -114,7 +114,7 @@ describe('AgentManager', () => {
 
     describe('two agent factories', () => {
       it('never provide the same Agent instance even if they use the same type', () => {
-        const agentManager = new AgentManager(logger);
+        const agentManager = new AgentManager(logger, { dnsCacheTtlInSeconds: 0 });
         const agentFactory1 = agentManager.getAgentFactory();
         const agentFactory2 = agentManager.getAgentFactory();
         const agent1 = agentFactory1({ url: new URL('http://elastic-node-1:9200') });
@@ -126,7 +126,7 @@ describe('AgentManager', () => {
 
   describe('#getAgentsStats()', () => {
     it('returns the stats of the agents', () => {
-      const agentManager = new AgentManager(logger);
+      const agentManager = new AgentManager(logger, { dnsCacheTtlInSeconds: 0 });
       const metrics: ElasticsearchClientsMetrics = {
         totalQueuedRequests: 0,
         totalIdleSockets: 100,
@@ -138,7 +138,7 @@ describe('AgentManager', () => {
     });
 
     it('warns when there are queued requests (requests unassigned to any socket)', () => {
-      const agentManager = new AgentManager(logger);
+      const agentManager = new AgentManager(logger, { dnsCacheTtlInSeconds: 0 });
       const metrics: ElasticsearchClientsMetrics = {
         totalQueuedRequests: 2,
         totalIdleSockets: 100, // There may be idle sockets when many clients are initialized. It should not be taken as an indicator of health.

packages/core/elasticsearch/core-elasticsearch-client-server-internal/src/agent_manager.ts

@@ -8,6 +8,7 @@
 
 import { Agent as HttpAgent, type AgentOptions } from 'http';
 import { Agent as HttpsAgent } from 'https';
+import CacheableLookup from 'cacheable-lookup';
 import type { ConnectionOptions, HttpAgentOptions } from '@elastic/elasticsearch';
 import type { Logger } from '@kbn/logging';
 import type { ElasticsearchClientsMetrics } from '@kbn/core-metrics-server';
@@ -22,6 +23,14 @@ export interface AgentFactoryProvider {
   getAgentFactory(agentOptions?: HttpAgentOptions): AgentFactory;
 }
 
+export interface AgentManagerOptions {
+  /**
+   * The maximum number of seconds to retain the DNS lookup resolutions.
+   * Set to 0 to disable the cache (default Node.js behavior)
+   */
+  dnsCacheTtlInSeconds: number;
+}
+
 /**
  * Exposes the APIs to fetch stats of the existing agents.
  */
@@ -45,9 +54,16 @@ export interface AgentStatsProvider {
  **/
 export class AgentManager implements AgentFactoryProvider, AgentStatsProvider {
   private readonly agents: Set<HttpAgent>;
+  private readonly cacheableLookup?: CacheableLookup;
 
-  constructor(private readonly logger: Logger) {
+  constructor(private readonly logger: Logger, options: AgentManagerOptions) {
     this.agents = new Set();
+    // Use DNS caching to avoid too many repetitive (and CPU-blocking) dns.lookup calls
+    if (options.dnsCacheTtlInSeconds > 0) {
+      this.cacheableLookup = new CacheableLookup({
+        maxTtl: options.dnsCacheTtlInSeconds,
+      });
+    }
   }
 
   public getAgentFactory(agentOptions?: AgentOptions): AgentFactory {
@@ -63,6 +79,7 @@ export class AgentManager implements AgentFactoryProvider, AgentStatsProvider {
           httpsAgent = new HttpsAgent(config);
           this.agents.add(httpsAgent);
           dereferenceOnDestroy(this.agents, httpsAgent);
+          this.cacheableLookup?.install(httpsAgent);
         }
 
         return httpsAgent;
@@ -72,6 +89,7 @@ export class AgentManager implements AgentFactoryProvider, AgentStatsProvider {
         httpAgent = new HttpAgent(agentOptions);
         this.agents.add(httpAgent);
         dereferenceOnDestroy(this.agents, httpAgent);
+        this.cacheableLookup?.install(httpAgent);
       }
 
       return httpAgent;

packages/core/elasticsearch/core-elasticsearch-client-server-internal/src/client_config.test.ts

@@ -25,6 +25,7 @@ const createConfig = (
     sniffInterval: false,
     requestHeadersWhitelist: ['authorization'],
     hosts: ['http://localhost:80'],
+    dnsCacheTtlInSeconds: 0,
     ...parts,
   };
 };

packages/core/elasticsearch/core-elasticsearch-client-server-internal/src/cluster_client.test.ts

@@ -34,6 +34,7 @@ const createConfig = (
     requestHeadersWhitelist: ['authorization'],
     customHeaders: {},
     hosts: ['http://localhost'],
+    dnsCacheTtlInSeconds: 0,
     ...parts,
   };
 };
@@ -57,7 +58,7 @@ describe('ClusterClient', () => {
     logger = loggingSystemMock.createLogger();
     internalClient = createClient();
     scopedClient = createClient();
-    agentFactoryProvider = new AgentManager(logger);
+    agentFactoryProvider = new AgentManager(logger, { dnsCacheTtlInSeconds: 0 });
 
     authHeaders = httpServiceMock.createAuthHeaderStorage();
     authHeaders.get.mockImplementation(() => ({

packages/core/elasticsearch/core-elasticsearch-client-server-internal/src/configure_client.test.ts

@@ -54,7 +54,7 @@ describe('configureClient', () => {
     config = createFakeConfig();
     parseClientOptionsMock.mockReturnValue({});
     ClientMock.mockImplementation(() => createFakeClient());
-    agentFactoryProvider = new AgentManager(logger);
+    agentFactoryProvider = new AgentManager(logger, { dnsCacheTtlInSeconds: 0 });
   });
 
   afterEach(() => {

packages/core/elasticsearch/core-elasticsearch-server-internal/src/elasticsearch_config.test.ts

@@ -33,6 +33,7 @@ test('set correct defaults', () => {
       "apisToRedactInLogs": Array [],
       "compression": false,
       "customHeaders": Object {},
+      "dnsCacheTtlInSeconds": 0,
       "healthCheckDelay": "PT2.5S",
       "healthCheckStartupDelay": "PT0.5S",
       "hosts": Array [

packages/core/elasticsearch/core-elasticsearch-server-internal/src/elasticsearch_config.ts

@@ -186,6 +186,7 @@ export const configSchema = schema.object({
     }),
     { defaultValue: [] }
   ),
+  dnsCacheTtlInSeconds: schema.number({ defaultValue: 0, min: 0, max: Infinity }),
 });
 
 const deprecations: ConfigDeprecationProvider = () => [
@@ -427,6 +428,12 @@ export class ElasticsearchConfig implements IElasticsearchConfig {
    */
   public readonly apisToRedactInLogs: ElasticsearchApiToRedactInLogs[];
 
+  /**
+   * The maximum number of seconds to retain the DNS lookup resolutions.
+   * Set to 0 to disable the cache (default Node.js behavior)
+   */
+  public readonly dnsCacheTtlInSeconds: number;
+
   constructor(rawConfig: ElasticsearchConfigType) {
     this.ignoreVersionMismatch = rawConfig.ignoreVersionMismatch;
     this.apiVersion = rawConfig.apiVersion;
@@ -452,6 +459,7 @@ export class ElasticsearchConfig implements IElasticsearchConfig {
     this.compression = rawConfig.compression;
     this.skipStartupConnectionCheck = rawConfig.skipStartupConnectionCheck;
     this.apisToRedactInLogs = rawConfig.apisToRedactInLogs;
+    this.dnsCacheTtlInSeconds = rawConfig.dnsCacheTtlInSeconds;
 
     const { alwaysPresentCertificate, verificationMode } = rawConfig.ssl;
     const { key, keyPassphrase, certificate, certificateAuthorities } = readKeyAndCerts(rawConfig);

packages/core/elasticsearch/core-elasticsearch-server-internal/src/elasticsearch_service.ts

@@ -61,15 +61,14 @@ export class ElasticsearchService
   private client?: ClusterClient;
   private clusterInfo$?: Observable<ClusterInfo>;
   private unauthorizedErrorHandler?: UnauthorizedErrorHandler;
-  private agentManager: AgentManager;
+  private agentManager?: AgentManager;
 
   constructor(private readonly coreContext: CoreContext) {
     this.kibanaVersion = coreContext.env.packageInfo.version;
     this.log = coreContext.logger.get('elasticsearch-service');
     this.config$ = coreContext.configService
       .atPath<ElasticsearchConfigType>('elasticsearch')
       .pipe(map((rawConfig) => new ElasticsearchConfig(rawConfig)));
-    this.agentManager = new AgentManager(this.log.get('agent-manager'));
   }
 
   public async preboot(): Promise<InternalElasticsearchServicePreboot> {
@@ -93,6 +92,8 @@ export class ElasticsearchService
 
     const config = await firstValueFrom(this.config$);
 
+    const agentManager = this.getAgentManager(config);
+
     this.authHeaders = deps.http.authRequestHeaders;
     this.executionContextClient = deps.executionContext;
     this.client = this.createClusterClient('data', config);
@@ -125,7 +126,7 @@ export class ElasticsearchService
         this.unauthorizedErrorHandler = handler;
       },
       agentStatsProvider: {
-        getAgentsStats: this.agentManager.getAgentsStats.bind(this.agentManager),
+        getAgentsStats: agentManager.getAgentsStats.bind(agentManager),
       },
     };
   }
@@ -218,8 +219,15 @@ export class ElasticsearchService
       authHeaders: this.authHeaders,
       getExecutionContext: () => this.executionContextClient?.getAsHeader(),
       getUnauthorizedErrorHandler: () => this.unauthorizedErrorHandler,
-      agentFactoryProvider: this.agentManager,
+      agentFactoryProvider: this.getAgentManager(baseConfig),
       kibanaVersion: this.kibanaVersion,
     });
   }
+
+  private getAgentManager({ dnsCacheTtlInSeconds }: ElasticsearchClientConfig): AgentManager {
+    if (!this.agentManager) {
+      this.agentManager = new AgentManager(this.log.get('agent-manager'), { dnsCacheTtlInSeconds });
+    }
+    return this.agentManager;
+  }
 }

packages/core/elasticsearch/core-elasticsearch-server/src/client/client_config.ts

@@ -50,6 +50,7 @@ export interface ElasticsearchClientConfig {
   caFingerprint?: string;
   ssl?: ElasticsearchClientSslConfig;
   apisToRedactInLogs?: ElasticsearchApiToRedactInLogs[];
+  dnsCacheTtlInSeconds: number;
 }
 
 /**

packages/core/elasticsearch/core-elasticsearch-server/src/elasticsearch_config.ts

@@ -149,6 +149,12 @@ export interface IElasticsearchConfig {
    * Extends the list of APIs that should be redacted in logs.
    */
   readonly apisToRedactInLogs: ElasticsearchApiToRedactInLogs[];
+
+  /**
+   * The maximum number of seconds to retain the DNS lookup resolutions.
+   * Set to 0 to disable the cache (default Node.js behavior)
+   */
+  readonly dnsCacheTtlInSeconds: number;
 }
 
 /**

packages/core/metrics/core-metrics-server-internal/src/ops_metrics_collector.test.ts

@@ -29,7 +29,7 @@ describe('OpsMetricsCollector', () => {
 
   beforeEach(() => {
     const hapiServer = httpServiceMock.createInternalSetupContract().server;
-    const agentManager = new AgentManager(loggerMock.create());
+    const agentManager = new AgentManager(loggerMock.create(), { dnsCacheTtlInSeconds: 0 });
     collector = new OpsMetricsCollector(hapiServer, agentManager, { logger: loggerMock.create() });
 
     mockOsCollector.collect.mockResolvedValue('osMetrics');

packages/core/test-helpers/core-test-helpers-model-versions/src/test_bed/test_kit.ts

@@ -202,7 +202,8 @@ const getElasticsearchClient = async (
     logger: loggerFactory.get('elasticsearch'),
     type: 'data',
     agentFactoryProvider: new AgentManager(
-      loggerFactory.get('elasticsearch-service', 'agent-manager')
+      loggerFactory.get('elasticsearch-service', 'agent-manager'),
+      { dnsCacheTtlInSeconds: 0 }
     ),
     kibanaVersion,
   });

src/cli_setup/utils.ts

@@ -49,7 +49,9 @@ export const elasticsearch = new ElasticsearchService(logger, kibanaPackageJson.
         logger,
         type,
         // we use an independent AgentManager for cli_setup, no need to track performance of this one
-        agentFactoryProvider: new AgentManager(logger.get('agent-manager')),
+        agentFactoryProvider: new AgentManager(logger.get('agent-manager'), {
+          dnsCacheTtlInSeconds: 0,
+        }),
         kibanaVersion: kibanaPackageJson.version,
       });
     },

src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts

@@ -266,7 +266,8 @@ const getElasticsearchClient = async (
     logger: loggerFactory.get('elasticsearch'),
     type: 'data',
     agentFactoryProvider: new AgentManager(
-      loggerFactory.get('elasticsearch-service', 'agent-manager')
+      loggerFactory.get('elasticsearch-service', 'agent-manager'),
+      { dnsCacheTtlInSeconds: 0 }
     ),
     kibanaVersion,
   });

x-pack/plugins/monitoring/server/config.test.ts

@@ -59,6 +59,7 @@ describe('config schema', () => {
             "apisToRedactInLogs": Array [],
             "compression": false,
             "customHeaders": Object {},
+            "dnsCacheTtlInSeconds": 0,
             "healthCheck": Object {
               "delay": "PT2.5S",
               "startupDelay": "PT0.5S",

yarn.lock

@@ -13461,6 +13461,11 @@ cache-base@^1.0.1:
     union-value "^1.0.0"
     unset-value "^1.0.0"
 
+cacheable-lookup@6:
+  version "6.1.0"
+  resolved "https://registry.yarnpkg.com/cacheable-lookup/-/cacheable-lookup-6.1.0.tgz#0330a543471c61faa4e9035db583aad753b36385"
+  integrity sha512-KJ/Dmo1lDDhmW2XDPMo+9oiy/CeqosPguPCrgcVzKyZrL6pM1gU2GmPY/xo6OQPTUaA/c0kwHuywB4E6nmT9ww==
+
 cacheable-lookup@^5.0.3:
   version "5.0.3"
   resolved "https://registry.yarnpkg.com/cacheable-lookup/-/cacheable-lookup-5.0.3.tgz#049fdc59dffdd4fc285e8f4f82936591bd59fec3"