Custom healthcheck with filters (opensearch-project#2232)

Enable filtering with custom health checks based on node attributes:
```
opensearch.optimizedHealthcheck.filters: {
  attribute_key: "attribute_value",
}
```

Also, fixes issue that expects the response to array when it was a
dictionary.

Issue:
opensearch-project#2214
opensearch-project#2203

Signed-off-by: Kawika Avilla <[email protected]>

config/opensearch_dashboards.yml

@@ -40,7 +40,10 @@
 # This node attribute should assign all nodes of the same cluster an integer value that increments with each new cluster that is spun up
 # e.g. in opensearch.yml file you would set the value to a setting using node.attr.cluster_id: 
 # Should only be enabled if there is a corresponding node attribute created in your OpenSearch config that matches the value here
-#opensearch.optimizedHealthcheckId: "cluster_id"
+#opensearch.optimizedHealthcheck.id: "cluster_id"
+#opensearch.optimizedHealthcheck.filters: {
+#  attribute_key: "attribute_value",
+#}
 
 # If your OpenSearch is protected with basic authentication, these settings provide
 # the username and password that the OpenSearch Dashboards server uses to perform maintenance on the OpenSearch Dashboards

src/core/server/opensearch/opensearch_config.test.ts

@@ -80,7 +80,7 @@ test('set correct defaults', () => {
       ],
       "ignoreVersionMismatch": false,
       "logQueries": false,
-      "optimizedHealthcheckId": undefined,
+      "optimizedHealthcheck": undefined,
       "password": undefined,
       "pingTimeout": "PT30S",
       "requestHeadersWhitelist": Array [
@@ -488,11 +488,20 @@ describe('deprecations', () => {
     `);
   });
 
-  it('logs a warning if elasticsearch.optimizedHealthcheckId is set and opensearch.optimizedHealthcheckId is not', () => {
+  it('logs a warning if elasticsearch.optimizedHealthcheckId is set and opensearch.optimizedHealthcheck.id is not', () => {
     const { messages } = applyLegacyDeprecations({ optimizedHealthcheckId: '' });
     expect(messages).toMatchInlineSnapshot(`
       Array [
-        "\\"elasticsearch.optimizedHealthcheckId\\" is deprecated and has been replaced by \\"opensearch.optimizedHealthcheckId\\"",
+        "\\"elasticsearch.optimizedHealthcheckId\\" is deprecated and has been replaced by \\"opensearch.optimizedHealthcheck.id\\"",
+      ]
+    `);
+  });
+
+  it('logs a warning if opensearch.optimizedHealthcheckId is set and opensearch.optimizedHealthcheck.id is not', () => {
+    const { messages } = applyOpenSearchDeprecations({ optimizedHealthcheckId: '' });
+    expect(messages).toMatchInlineSnapshot(`
+      Array [
+        "\\"opensearch.optimizedHealthcheckId\\" is deprecated and has been replaced by \\"opensearch.optimizedHealthcheck.id\\"",
       ]
     `);
   });

src/core/server/opensearch/opensearch_config.ts

@@ -84,7 +84,14 @@ export const configSchema = schema.object({
   requestTimeout: schema.duration({ defaultValue: '30s' }),
   pingTimeout: schema.duration({ defaultValue: schema.siblingRef('requestTimeout') }),
   logQueries: schema.boolean({ defaultValue: false }),
-  optimizedHealthcheckId: schema.maybe(schema.string()),
+  optimizedHealthcheck: schema.maybe(
+    schema.object({
+      id: schema.string(),
+      filters: schema.maybe(
+        schema.recordOf(schema.string(), schema.string(), { defaultValue: {} })
+      ),
+    })
+  ),
   ssl: schema.object(
     {
       verificationMode: schema.oneOf(
@@ -148,7 +155,8 @@ const deprecations: ConfigDeprecationProvider = ({ renameFromRoot }) => [
   renameFromRoot('elasticsearch.requestTimeout', 'opensearch.requestTimeout'),
   renameFromRoot('elasticsearch.pingTimeout', 'opensearch.pingTimeout'),
   renameFromRoot('elasticsearch.logQueries', 'opensearch.logQueries'),
-  renameFromRoot('elasticsearch.optimizedHealthcheckId', 'opensearch.optimizedHealthcheckId'),
+  renameFromRoot('elasticsearch.optimizedHealthcheckId', 'opensearch.optimizedHealthcheck.id'),
+  renameFromRoot('opensearch.optimizedHealthcheckId', 'opensearch.optimizedHealthcheck.id'),
   renameFromRoot('elasticsearch.ssl', 'opensearch.ssl'),
   renameFromRoot('elasticsearch.apiVersion', 'opensearch.apiVersion'),
   renameFromRoot('elasticsearch.healthCheck', 'opensearch.healthCheck'),
@@ -216,7 +224,7 @@ export class OpenSearchConfig {
    * Specifies whether Dashboards should only query the local OpenSearch node when
    * all nodes in the cluster have the same node attribute value
    */
-  public readonly optimizedHealthcheckId?: string;
+  public readonly optimizedHealthcheck?: OpenSearchConfigType['optimizedHealthcheck'];
 
   /**
    * Hosts that the client will connect to. If sniffing is enabled, this list will
@@ -297,7 +305,7 @@ export class OpenSearchConfig {
     this.ignoreVersionMismatch = rawConfig.ignoreVersionMismatch;
     this.apiVersion = rawConfig.apiVersion;
     this.logQueries = rawConfig.logQueries;
-    this.optimizedHealthcheckId = rawConfig.optimizedHealthcheckId;
+    this.optimizedHealthcheck = rawConfig.optimizedHealthcheck;
     this.hosts = Array.isArray(rawConfig.hosts) ? rawConfig.hosts : [rawConfig.hosts];
     this.requestHeadersWhitelist = Array.isArray(rawConfig.requestHeadersWhitelist)
       ? rawConfig.requestHeadersWhitelist

src/core/server/opensearch/opensearch_service.ts

@@ -95,7 +95,7 @@ export class OpenSearchService
 
     const opensearchNodesCompatibility$ = pollOpenSearchNodesVersion({
       internalClient: this.client.asInternalUser,
-      optimizedHealthcheckId: config.optimizedHealthcheckId,
+      optimizedHealthcheck: config.optimizedHealthcheck,
       log: this.log,
       ignoreVersionMismatch: config.ignoreVersionMismatch,
       opensearchVersionCheckInterval: config.healthCheckDelay.asMilliseconds(),

src/core/server/opensearch/version_check/ensure_opensearch_version.test.ts

@@ -67,6 +67,35 @@ function createNodes(...versions: string[]): NodesInfo {
   return { nodes };
 }
 
+function createNodesWithAttribute(
+  targetId: string,
+  filterId: string,
+  targetAttributeValue: string,
+  filterAttributeValue: string,
+  ...versions: string[]
+): NodesInfo {
+  const nodes = {} as any;
+  versions
+    .map((version, i) => {
+      return {
+        version,
+        http: {
+          publish_address: 'http_address',
+        },
+        ip: 'ip',
+        attributes: {
+          cluster_id: i % 2 === 0 ? targetId : filterId,
+          custom_attribute: i % 2 === 0 ? targetAttributeValue : filterAttributeValue,
+        },
+      };
+    })
+    .forEach((node, i) => {
+      nodes[`node-${i}`] = node;
+    });
+
+  return { nodes };
+}
+
 describe('mapNodesVersionCompatibility', () => {
   function createNodesInfoWithoutHTTP(version: string): NodesInfo {
     return { nodes: { 'node-without-http': { version, ip: 'ip' } } } as any;
@@ -180,7 +209,7 @@ describe('pollOpenSearchNodesVersion', () => {
 
     pollOpenSearchNodesVersion({
       internalClient,
-      optimizedHealthcheckId: 'cluster_id',
+      optimizedHealthcheck: { id: 'cluster_id' },
       opensearchVersionCheckInterval: 1,
       ignoreVersionMismatch: false,
       opensearchDashboardsVersion: OPENSEARCH_DASHBOARDS_VERSION,
@@ -204,7 +233,104 @@ describe('pollOpenSearchNodesVersion', () => {
 
     pollOpenSearchNodesVersion({
       internalClient,
-      optimizedHealthcheckId: 'cluster_id',
+      optimizedHealthcheck: { id: 'cluster_id' },
+      opensearchVersionCheckInterval: 1,
+      ignoreVersionMismatch: false,
+      opensearchDashboardsVersion: OPENSEARCH_DASHBOARDS_VERSION,
+      log: mockLogger,
+    })
+      .pipe(take(1))
+      .subscribe({
+        next: (result) => {
+          expect(result).toEqual(
+            mapNodesVersionCompatibility(nodes, OPENSEARCH_DASHBOARDS_VERSION, false)
+          );
+        },
+        complete: done,
+        error: done,
+      });
+  });
+
+  it('returns compatibility results and isCompatible=true with filters', (done) => {
+    expect.assertions(2);
+    const target = {
+      id: '0',
+      attribute: 'foo',
+    };
+    const filter = {
+      id: '1',
+      attribute: 'bar',
+    };
+
+    // will filter out every odd index
+    const nodes = createNodesWithAttribute(
+      target.id,
+      filter.id,
+      target.attribute,
+      filter.attribute,
+      '5.1.0',
+      '6.2.0',
+      '5.1.0',
+      '5.1.1-Beta1'
+    );
+
+    // @ts-expect-error we need to return an incompatible type to use the testScheduler here
+    internalClient.cluster.state.mockReturnValueOnce({ body: nodes });
+
+    nodeInfosSuccessOnce(nodes);
+
+    pollOpenSearchNodesVersion({
+      internalClient,
+      optimizedHealthcheck: { id: target.id, filters: { custom_attribute: filter.attribute } },
+      opensearchVersionCheckInterval: 1,
+      ignoreVersionMismatch: false,
+      opensearchDashboardsVersion: OPENSEARCH_DASHBOARDS_VERSION,
+      log: mockLogger,
+    })
+      .pipe(take(1))
+      .subscribe({
+        next: (result) => {
+          expect(result).toEqual(
+            mapNodesVersionCompatibility(nodes, OPENSEARCH_DASHBOARDS_VERSION, false)
+          );
+          expect(result.isCompatible).toBe(true);
+        },
+        complete: done,
+        error: done,
+      });
+  });
+
+  it('returns compatibility results and isCompatible=false with filters', (done) => {
+    expect.assertions(2);
+    const target = {
+      id: '0',
+      attribute: 'foo',
+    };
+    const filter = {
+      id: '1',
+      attribute: 'bar',
+    };
+
+    // will filter out every odd index
+    const nodes = createNodesWithAttribute(
+      target.id,
+      filter.id,
+      target.attribute,
+      filter.attribute,
+      '5.1.0',
+      '5.1.0',
+      '6.2.0',
+      '5.1.1-Beta1'
+    );
+
+    // @ts-expect-error we need to return an incompatible type to use the testScheduler here
+    internalClient.cluster.state.mockReturnValueOnce({ body: nodes });
+
+    nodeInfosSuccessOnce(nodes);
+
+    pollOpenSearchNodesVersion({
+      internalClient,
+      optimizedHealthcheck: { id: target.id, filters: { custom_attribute: filter.attribute } },
       opensearchVersionCheckInterval: 1,
       ignoreVersionMismatch: false,
       opensearchDashboardsVersion: OPENSEARCH_DASHBOARDS_VERSION,
@@ -216,6 +342,7 @@ describe('pollOpenSearchNodesVersion', () => {
           expect(result).toEqual(
             mapNodesVersionCompatibility(nodes, OPENSEARCH_DASHBOARDS_VERSION, false)
           );
+          expect(result.isCompatible).toBe(false);
         },
         complete: done,
         error: done,
@@ -233,7 +360,7 @@ describe('pollOpenSearchNodesVersion', () => {
 
     pollOpenSearchNodesVersion({
       internalClient,
-      optimizedHealthcheckId: 'cluster_id',
+      optimizedHealthcheck: { id: 'cluster_id' },
       opensearchVersionCheckInterval: 1,
       ignoreVersionMismatch: false,
       opensearchDashboardsVersion: OPENSEARCH_DASHBOARDS_VERSION,
@@ -269,7 +396,7 @@ describe('pollOpenSearchNodesVersion', () => {
 
       const opensearchNodesCompatibility$ = pollOpenSearchNodesVersion({
         internalClient,
-        optimizedHealthcheckId: 'cluster_id',
+        optimizedHealthcheck: { id: 'cluster_id' },
         opensearchVersionCheckInterval: 100,
         ignoreVersionMismatch: false,
         opensearchDashboardsVersion: OPENSEARCH_DASHBOARDS_VERSION,
@@ -309,7 +436,7 @@ describe('pollOpenSearchNodesVersion', () => {
 
       const opensearchNodesCompatibility$ = pollOpenSearchNodesVersion({
         internalClient,
-        optimizedHealthcheckId: 'cluster_id',
+        optimizedHealthcheck: { id: 'cluster_id' },
         opensearchVersionCheckInterval: 10,
         ignoreVersionMismatch: false,
         opensearchDashboardsVersion: OPENSEARCH_DASHBOARDS_VERSION,