[Backport 2.x] Add Index-based adaptor for integrations

server/adaptors/integrations/__test__/json_repository.test.ts

@@ -42,10 +42,9 @@ describe('The Local Serialized Catalog', () => {
 
   it('Should pass deep validation for all serialized integrations', async () => {
     const serialized = await fetchSerializedIntegrations();
-    const repository = new TemplateManager(
-      '.',
-      new JsonCatalogDataAdaptor(serialized.value as SerializedIntegration[])
-    );
+    const repository = new TemplateManager([
+      new JsonCatalogDataAdaptor(serialized.value as SerializedIntegration[]),
+    ]);
 
     for (const integ of await repository.getIntegrationList()) {
       const validationResult = await deepCheck(integ);
@@ -55,10 +54,9 @@ describe('The Local Serialized Catalog', () => {
 
   it('Should correctly retrieve a logo', async () => {
     const serialized = await fetchSerializedIntegrations();
-    const repository = new TemplateManager(
-      '.',
-      new JsonCatalogDataAdaptor(serialized.value as SerializedIntegration[])
-    );
+    const repository = new TemplateManager([
+      new JsonCatalogDataAdaptor(serialized.value as SerializedIntegration[]),
+    ]);
     const integration = (await repository.getIntegration('nginx')) as IntegrationReader;
     const logoStatic = await integration.getStatic('logo.svg');
 
@@ -68,10 +66,9 @@ describe('The Local Serialized Catalog', () => {
 
   it('Should correctly retrieve a gallery image', async () => {
     const serialized = await fetchSerializedIntegrations();
-    const repository = new TemplateManager(
-      '.',
-      new JsonCatalogDataAdaptor(serialized.value as SerializedIntegration[])
-    );
+    const repository = new TemplateManager([
+      new JsonCatalogDataAdaptor(serialized.value as SerializedIntegration[]),
+    ]);
     const integration = (await repository.getIntegration('nginx')) as IntegrationReader;
     const logoStatic = await integration.getStatic('dashboard1.png');
 

server/adaptors/integrations/__test__/local_fs_repository.test.ts

@@ -12,6 +12,11 @@ import { IntegrationReader } from '../repository/integration_reader';
 import path from 'path';
 import * as fs from 'fs/promises';
 import { deepCheck } from '../repository/utils';
+import { FileSystemDataAdaptor } from '../repository/fs_data_adaptor';
+
+const repository: TemplateManager = new TemplateManager([
+  new FileSystemDataAdaptor(path.join(__dirname, '../__data__/repository')),
+]);
 
 describe('The local repository', () => {
   it('Should only contain valid integration directories or files.', async () => {
@@ -32,9 +37,6 @@ describe('The local repository', () => {
   });
 
   it('Should pass deep validation for all local integrations.', async () => {
-    const repository: TemplateManager = new TemplateManager(
-      path.join(__dirname, '../__data__/repository')
-    );
     const integrations: IntegrationReader[] = await repository.getIntegrationList();
     await Promise.all(
       integrations.map(async (i) => {
@@ -50,18 +52,12 @@ describe('The local repository', () => {
 
 describe('Local Nginx Integration', () => {
   it('Should serialize without errors', async () => {
-    const repository: TemplateManager = new TemplateManager(
-      path.join(__dirname, '../__data__/repository')
-    );
     const integration = await repository.getIntegration('nginx');
 
     await expect(integration?.serialize()).resolves.toHaveProperty('ok', true);
   });
 
   it('Should serialize to include the config', async () => {
-    const repository: TemplateManager = new TemplateManager(
-      path.join(__dirname, '../__data__/repository')
-    );
     const integration = await repository.getIntegration('nginx');
     const config = await integration!.getConfig();
     const serialized = await integration!.serialize();

server/adaptors/integrations/integrations_manager.ts

@@ -9,6 +9,8 @@
 import { SavedObject, SavedObjectsClientContract } from '../../../../../src/core/server/types';
 import { IntegrationInstanceBuilder } from './integrations_builder';
 import { TemplateManager } from './repository/repository';
+import { FileSystemDataAdaptor } from './repository/fs_data_adaptor';
+import { IndexDataAdaptor } from './repository/index_data_adaptor';
 
 export class IntegrationsManager implements IntegrationsAdaptor {
   client: SavedObjectsClientContract;
@@ -18,21 +20,25 @@
   constructor(client: SavedObjectsClientContract, repository?: TemplateManager) {
     this.client = client;
     this.repository =
-      repository ?? new TemplateManager(path.join(__dirname, '__data__/repository'));
+      repository ??
+      new TemplateManager([
+        new IndexDataAdaptor(this.client),
+        new FileSystemDataAdaptor(path.join(__dirname, '__data__/repository')),
+      ]);
     this.instanceBuilder = new IntegrationInstanceBuilder(this.client);
   }
 
   deleteIntegrationInstance = async (id: string): Promise<string[]> => {
-    let children: any;
+    let children: SavedObject<IntegrationInstance>;
     try {
       children = await this.client.get('integration-instance', id);
-    } catch (err: any) {
+    } catch (err) {
       return err.output?.statusCode === 404 ? Promise.resolve([id]) : Promise.reject(err);
     }
 
     const toDelete = children.attributes.assets
-      .filter((i: any) => i.assetId)
-      .map((i: any) => {
+      .filter((i: AssetReference) => i.assetId)
+      .map((i: AssetReference) => {
         return { id: i.assetId, type: i.assetType };
       });
     toDelete.push({ id, type: 'integration-instance' });
@@ -43,7 +49,7 @@
           try {
             await this.client.delete(asset.type, asset.id);
             return Promise.resolve(asset.id);
-          } catch (err: any) {
+          } catch (err) {
             addRequestToMetric('integrations', 'delete', err);
             return err.output?.statusCode === 404 ? Promise.resolve(asset.id) : Promise.reject(err);
           }
@@ -101,20 +107,22 @@
     query?: IntegrationInstanceQuery
   ): Promise<IntegrationInstanceResult> => {
     addRequestToMetric('integrations', 'get', 'count');
-    const result = await this.client.get('integration-instance', `${query!.id}`);
+    const result = (await this.client.get('integration-instance', `${query!.id}`)) as SavedObject<
+      IntegrationInstance
+    >;
     return Promise.resolve(this.buildInstanceResponse(result));
   };
 
   buildInstanceResponse = async (
-    savedObj: SavedObject<unknown>
+    savedObj: SavedObject<IntegrationInstance>
   ): Promise<IntegrationInstanceResult> => {
-    const assets: AssetReference[] | undefined = (savedObj.attributes as any)?.assets;
+    const assets: AssetReference[] | undefined = savedObj.attributes.assets;
     const status: string = assets ? await this.getAssetStatus(assets) : 'available';
 
     return {
       id: savedObj.id,
       status,
-      ...(savedObj.attributes as any),
+      ...savedObj.attributes,
     };
   };
 
@@ -124,7 +132,7 @@
         try {
           await this.client.get(asset.assetType, asset.assetId);
           return { id: asset.assetId, status: 'available' };
-        } catch (err: any) {
+        } catch (err) {
           const statusCode = err.output?.statusCode;
           if (statusCode && 400 <= statusCode && statusCode < 500) {
             return { id: asset.assetId, status: 'unavailable' };
@@ -166,7 +174,7 @@
       });
       const test = await this.client.create('integration-instance', result);
       return Promise.resolve({ ...result, id: test.id });
-    } catch (err: any) {
+    } catch (err) {
       addRequestToMetric('integrations', 'create', err);
       return Promise.reject({
         message: err.message,
@@ -213,7 +221,7 @@
     });
   };
 
-  getAssets = async (templateName: string): Promise<{ savedObjects?: any }> => {
+  getAssets = async (templateName: string): Promise<ParsedIntegrationAssets> => {
     const integration = await this.repository.getIntegration(templateName);
     if (integration === null) {
       return Promise.reject({

server/adaptors/integrations/repository/__test__/index_data_adaptor.test.ts

@@ -0,0 +1,105 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { IntegrationReader } from '../integration_reader';
+import { JsonCatalogDataAdaptor } from '../json_data_adaptor';
+import { TEST_INTEGRATION_CONFIG } from '../../../../../test/constants';
+import { savedObjectsClientMock } from '../../../../../../../src/core/server/mocks';
+import { IndexDataAdaptor } from '../index_data_adaptor';
+import { SavedObjectsClientContract } from '../../../../../../../src/core/server';
+
+// Simplified catalog for integration searching -- Do not use for full deserialization tests.
+const TEST_CATALOG_NO_SERIALIZATION: SerializedIntegration[] = [
+  {
+    ...(TEST_INTEGRATION_CONFIG as SerializedIntegration),
+    name: 'sample1',
+  },
+  {
+    ...(TEST_INTEGRATION_CONFIG as SerializedIntegration),
+    name: 'sample2',
+  },
+  {
+    ...(TEST_INTEGRATION_CONFIG as SerializedIntegration),
+    name: 'sample2',
+    version: '2.1.0',
+  },
+];
+
+// Copy of json_data_adaptor.test.ts with new reader type
+// Since implementation at time of writing is to defer to json adaptor
+describe('Index Data Adaptor', () => {
+  let mockClient: SavedObjectsClientContract;
+
+  beforeEach(() => {
+    mockClient = savedObjectsClientMock.create();
+    mockClient.find = jest.fn().mockResolvedValue({
+      saved_objects: TEST_CATALOG_NO_SERIALIZATION.map((item) => ({
+        attributes: item,
+      })),
+    });
+  });
+
+  it('Should correctly identify repository type', async () => {
+    const adaptor = new IndexDataAdaptor(mockClient);
+    await expect(adaptor.getDirectoryType()).resolves.toBe('repository');
+  });
+
+  it('Should correctly identify integration type after filtering', async () => {
+    const adaptor = new JsonCatalogDataAdaptor(TEST_CATALOG_NO_SERIALIZATION);
+    const joined = await adaptor.join('sample1');
+    await expect(joined.getDirectoryType()).resolves.toBe('integration');
+  });
+
+  it('Should correctly retrieve integration versions', async () => {
+    const adaptor = new IndexDataAdaptor(mockClient);
+    const versions = await adaptor.findIntegrationVersions('sample2');
+    expect((versions as { value: string[] }).value).toHaveLength(2);
+  });
+
+  it('Should correctly supply latest integration version for IntegrationReader', async () => {
+    const adaptor = new IndexDataAdaptor(mockClient);
+    const reader = new IntegrationReader('sample2', adaptor.join('sample2'));
+    const version = await reader.getLatestVersion();
+    expect(version).toBe('2.1.0');
+  });
+
+  it('Should find integration names', async () => {
+    const adaptor = new IndexDataAdaptor(mockClient);
+    const integResult = await adaptor.findIntegrations();
+    const integs = (integResult as { value: string[] }).value;
+    integs.sort();
+
+    expect(integs).toEqual(['sample1', 'sample2']);
+  });
+
+  it('Should reject any attempts to read a file with a type', async () => {
+    const adaptor = new IndexDataAdaptor(mockClient);
+    const result = await adaptor.readFile('logs-1.0.0.json', 'schemas');
+    await expect(result.error?.message).toBe(
+      'JSON adaptor does not support subtypes (isConfigLocalized: true)'
+    );
+  });
+
+  it('Should reject any attempts to read a raw file', async () => {
+    const adaptor = new JsonCatalogDataAdaptor(TEST_CATALOG_NO_SERIALIZATION);
+    const result = await adaptor.readFileRaw('logo.svg', 'static');
+    await expect(result.error?.message).toBe(
+      'JSON adaptor does not support raw files (isConfigLocalized: true)'
+    );
+  });
+
+  it('Should reject nested directory searching', async () => {
+    const adaptor = new JsonCatalogDataAdaptor(TEST_CATALOG_NO_SERIALIZATION);
+    const result = await adaptor.findIntegrations('sample1');
+    await expect(result.error?.message).toBe(
+      'Finding integrations for custom dirs not supported for JSONreader'
+    );
+  });
+
+  it('Should report unknown directory type if integration list is empty', async () => {
+    const adaptor = new JsonCatalogDataAdaptor([]);
+    await expect(adaptor.getDirectoryType()).resolves.toBe('unknown');
+  });
+});

server/adaptors/integrations/repository/__test__/json_data_adaptor.test.ts

@@ -8,6 +8,7 @@ import { IntegrationReader } from '../integration_reader';
 import path from 'path';
 import { JsonCatalogDataAdaptor } from '../json_data_adaptor';
 import { TEST_INTEGRATION_CONFIG } from '../../../../../test/constants';
+import { FileSystemDataAdaptor } from '../fs_data_adaptor';
 
 // Simplified catalog for integration searching -- Do not use for full deserialization tests.
 const TEST_CATALOG_NO_SERIALIZATION: SerializedIntegration[] = [
@@ -28,9 +29,9 @@ const TEST_CATALOG_NO_SERIALIZATION: SerializedIntegration[] = [
 
 describe('JSON Data Adaptor', () => {
   it('Should be able to deserialize a serialized integration', async () => {
-    const repository: TemplateManager = new TemplateManager(
-      path.join(__dirname, '../../__data__/repository')
-    );
+    const repository: TemplateManager = new TemplateManager([
+      new FileSystemDataAdaptor(path.join(__dirname, '../../__data__/repository')),
+    ]);
     const fsIntegration: IntegrationReader = (await repository.getIntegration('nginx'))!;
     const fsConfig = await fsIntegration.getConfig();
     const serialized = await fsIntegration.serialize();
@@ -112,4 +113,13 @@ describe('JSON Data Adaptor', () => {
     const adaptor = new JsonCatalogDataAdaptor([]);
     await expect(adaptor.getDirectoryType()).resolves.toBe('unknown');
   });
+
+  // Bug: a previous regex for version finding counted the `8` in `k8s-1.0.0.json` as the version
+  it('Should correctly read a config with a number in the name', async () => {
+    const adaptor = new JsonCatalogDataAdaptor(TEST_CATALOG_NO_SERIALIZATION);
+    await expect(adaptor.readFile('sample2-2.1.0.json')).resolves.toMatchObject({
+      ok: true,
+      value: TEST_CATALOG_NO_SERIALIZATION[2],
+    });
+  });
 });

server/adaptors/integrations/repository/__test__/repository.test.ts

@@ -8,14 +8,15 @@ import { TemplateManager } from '../repository';
 import { IntegrationReader } from '../integration_reader';
 import { Dirent, Stats } from 'fs';
 import path from 'path';
+import { FileSystemDataAdaptor } from '../fs_data_adaptor';
 
 jest.mock('fs/promises');
 
 describe('Repository', () => {
   let repository: TemplateManager;
 
   beforeEach(() => {
-    repository = new TemplateManager('path/to/directory');
+    repository = new TemplateManager([new FileSystemDataAdaptor('path/to/directory')]);
   });
 
   afterEach(() => {

server/adaptors/integrations/repository/fs_data_adaptor.ts

@@ -21,7 +21,7 @@ const safeIsDirectory = async (maybeDirectory: string): Promise<boolean> => {
  * A CatalogDataAdaptor that reads from the local filesystem.
  * Used to read default Integrations shipped in the in-product catalog at `__data__`.
  */
-export class FileSystemCatalogDataAdaptor implements CatalogDataAdaptor {
+export class FileSystemDataAdaptor implements CatalogDataAdaptor {
   isConfigLocalized = false;
   directory: string;
 
@@ -131,7 +131,7 @@ export class FileSystemCatalogDataAdaptor implements CatalogDataAdaptor {
     return hasSchemas ? 'integration' : 'repository';
   }
 
-  join(filename: string): FileSystemCatalogDataAdaptor {
-    return new FileSystemCatalogDataAdaptor(path.join(this.directory, filename));
+  join(filename: string): FileSystemDataAdaptor {
+    return new FileSystemDataAdaptor(path.join(this.directory, filename));
   }
 }