[Multiple DataSource] Add support for SigV4 authentication

Signed-off-by: Su <[email protected]>

CHANGELOG.md

@@ -58,6 +58,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [Optimizer] Increase timeout waiting for the exiting of an optimizer worker ([#3193](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3193))
 - [Data] Update `createAggConfig` so that newly created configs can be added to beginning of `aggConfig` array ([#3160](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3160))
 - Add disablePrototypePoisoningProtection configuration to prevent JS client from erroring when cluster utilizes JS reserved words ([#2992](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2992))
+- [Multiple DataSource] Add support for SigV4 authentication ([#3058](https://github.com/opensearch-project/OpenSearch-Dashboards/issues/3058))
 
 ### 🐛 Bug Fixes
 

config/opensearch_dashboards.yml

@@ -231,7 +231,7 @@
 
 # Set the value of this setting to true to enable the experimental multiple data source
 # support feature. Use with caution.
-#data_source.enabled: false
+data_source.enabled: true
 # Set the value of these settings to customize crypto materials to encryption saved credentials
 # in data sources.
 #data_source.encryption.wrappingKeyName: 'changeme'

package.json

@@ -133,7 +133,7 @@
     "@hapi/podium": "^4.1.3",
     "@hapi/vision": "^6.1.0",
     "@hapi/wreck": "^17.1.0",
-    "@opensearch-project/opensearch": "^1.1.0",
+    "@opensearch-project/opensearch": "^2.1.0",
     "@osd/ace": "1.0.0",
     "@osd/analytics": "1.0.0",
     "@osd/apm-config-loader": "1.0.0",
@@ -166,6 +166,7 @@
     "dns-sync": "^0.2.1",
     "elastic-apm-node": "^3.7.0",
     "elasticsearch": "^16.7.0",
+    "http-aws-es": "6.0.0",
     "execa": "^4.0.2",
     "expiry-js": "0.1.7",
     "fast-deep-equal": "^3.1.1",
@@ -334,6 +335,7 @@
     "@types/zen-observable": "^0.8.0",
     "@typescript-eslint/eslint-plugin": "^3.10.0",
     "@typescript-eslint/parser": "^3.10.0",
+    "@types/http-aws-es": "6.0.2",
     "angular-aria": "^1.8.0",
     "angular-mocks": "^1.8.2",
     "angular-recursion": "^1.0.5",

packages/osd-opensearch/package.json

@@ -12,7 +12,7 @@
     "osd:watch": "node scripts/build --watch"
   },
   "dependencies": {
-    "@opensearch-project/opensearch": "^1.1.0",
+    "@opensearch-project/opensearch": "^2.1.0",
     "@osd/dev-utils": "1.0.0",
     "abort-controller": "^3.0.0",
     "chalk": "^4.1.0",

src/dev/jest/config.js

@@ -52,6 +52,8 @@ export default {
   moduleNameMapper: {
     '@elastic/eui$': '<rootDir>/node_modules/@elastic/eui/test-env',
     '@elastic/eui/lib/(.*)?': '<rootDir>/node_modules/@elastic/eui/test-env/$1',
+    '@opensearch-project/opensearch/aws':
+      '<rootDir>/node_modules/@opensearch-project/opensearch/lib/aws',
     '^src/plugins/(.*)': '<rootDir>/src/plugins/$1',
     '^test_utils/(.*)': '<rootDir>/src/test_utils/public/$1',
     '^fixtures/(.*)': '<rootDir>/src/fixtures/$1',

src/plugins/data_source/common/data_sources/types.ts

@@ -11,8 +11,15 @@ export interface DataSourceAttributes extends SavedObjectAttributes {
   endpoint: string;
   auth: {
     type: AuthType;
-    credentials: UsernamePasswordTypedContent | undefined;
+    credentials: UsernamePasswordTypedContent | SigV4Content | undefined;
   };
+  lastUpdatedTime?: string;
+}
+
+export interface SigV4Content extends SavedObjectAttributes {
+  accessKey: string;
+  secretKey: string;
+  region: string;
 }
 
 export interface UsernamePasswordTypedContent extends SavedObjectAttributes {
@@ -23,4 +30,5 @@ export interface UsernamePasswordTypedContent extends SavedObjectAttributes {
 export enum AuthType {
   NoAuth = 'no_auth',
   UsernamePasswordType = 'username_password',
+  SigV4 = 'sigv4',
 }

src/plugins/data_source/server/client/client_pool.ts

@@ -7,11 +7,12 @@ import { Client } from '@opensearch-project/opensearch';
 import { Client as LegacyClient } from 'elasticsearch';
 import LRUCache from 'lru-cache';
 import { Logger } from 'src/core/server';
+import { AuthType } from '../../common/data_sources';
 import { DataSourcePluginConfigType } from '../../config';
 
 export interface OpenSearchClientPoolSetup {
-  getClientFromPool: (id: string) => Client | LegacyClient | undefined;
-  addClientToPool: (endpoint: string, client: Client | LegacyClient) => void;
+  getClientFromPool: (endpoint: string, authType: AuthType) => Client | LegacyClient | undefined;
+  addClientToPool: (endpoint: string, authType: AuthType, client: Client | LegacyClient) => void;
 }
 
 /**
@@ -24,7 +25,8 @@ export class OpenSearchClientPool {
   // LRU cache
   //   key: data source endpoint
   //   value: OpenSearch client object | Legacy client object
-  private cache?: LRUCache<string, Client | LegacyClient>;
+  private clientCache?: LRUCache<string, Client | LegacyClient>;
+  private awsClientCache?: LRUCache<string, Client | LegacyClient>;
   private isClosed = false;
 
   constructor(private logger: Logger) {}
@@ -33,7 +35,7 @@ export class OpenSearchClientPool {
     const logger = this.logger;
     const { size } = config.clientPool;
 
-    this.cache = new LRUCache({
+    this.clientCache = new LRUCache({
       max: size,
       maxAge: 15 * 60 * 1000, // by default, TCP connection times out in 15 minutes
 
@@ -50,12 +52,35 @@ export class OpenSearchClientPool {
     });
     this.logger.info(`Created data source client pool of size ${size}`);
 
-    const getClientFromPool = (endpoint: string) => {
-      return this.cache!.get(endpoint);
+    // aws client specific pool
+    this.awsClientCache = new LRUCache({
+      max: size,
+      maxAge: 15 * 60 * 1000, // by default, TCP connection times out in 15 minutes
+
+      async dispose(endpoint, client) {
+        try {
+          await client.close();
+        } catch (error: any) {
+          // log and do nothing since we are anyways evicting the client object from cache
+          logger.warn(
+            `Error closing OpenSearch client when removing from aws client pool: ${error.message}`
+          );
+        }
+      },
+    });
+    this.logger.info(`Created data source aws client pool of size ${size}`);
+
+    const getClientFromPool = (endpoint: string, authType: AuthType) => {
+      const selectedCache = authType === AuthType.SigV4 ? this.awsClientCache : this.clientCache;
+
+      return selectedCache!.get(endpoint);
     };
 
-    const addClientToPool = (endpoint: string, client: Client | LegacyClient) => {
-      this.cache!.set(endpoint, client);
+    const addClientToPool = (endpoint: string, authType: string, client: Client | LegacyClient) => {
+      const selectedCache = authType === AuthType.SigV4 ? this.awsClientCache : this.clientCache;
+      if (!selectedCache?.has(endpoint)) {
+        return selectedCache!.set(endpoint, client);
+      }
     };
 
     return {
@@ -71,7 +96,10 @@ export class OpenSearchClientPool {
     if (this.isClosed) {
       return;
     }
-    await Promise.all(this.cache!.values().map((client) => client.close()));
+    await Promise.all([
+      ...this.clientCache!.values().map((client) => client.close()),
+      ...this.awsClientCache!.values().map((client) => client.close()),
+    ]);
     this.isClosed = true;
   }
 }