[6.x] Implement new platform plugin discovery. (#24256)

src/core/server/config/__snapshots__/env.test.ts.snap

@@ -16,7 +16,6 @@ Env {
   "configs": Array [
     "/some/other/path/some-kibana.yml",
   ],
-  "corePluginsDir": "/test/cwd/core_plugins",
   "homeDir": "/test/cwd",
   "isDevClusterMaster": false,
   "logDir": "/test/cwd/log",
@@ -51,7 +50,6 @@ Env {
   "configs": Array [
     "/some/other/path/some-kibana.yml",
   ],
-  "corePluginsDir": "/test/cwd/core_plugins",
   "homeDir": "/test/cwd",
   "isDevClusterMaster": false,
   "logDir": "/test/cwd/log",
@@ -85,7 +83,6 @@ Env {
   "configs": Array [
     "/test/cwd/config/kibana.yml",
   ],
-  "corePluginsDir": "/test/cwd/core_plugins",
   "homeDir": "/test/cwd",
   "isDevClusterMaster": true,
   "logDir": "/test/cwd/log",
@@ -119,7 +116,6 @@ Env {
   "configs": Array [
     "/some/other/path/some-kibana.yml",
   ],
-  "corePluginsDir": "/test/cwd/core_plugins",
   "homeDir": "/test/cwd",
   "isDevClusterMaster": false,
   "logDir": "/test/cwd/log",
@@ -153,7 +149,6 @@ Env {
   "configs": Array [
     "/some/other/path/some-kibana.yml",
   ],
-  "corePluginsDir": "/test/cwd/core_plugins",
   "homeDir": "/test/cwd",
   "isDevClusterMaster": false,
   "logDir": "/test/cwd/log",
@@ -187,7 +182,6 @@ Env {
   "configs": Array [
     "/some/other/path/some-kibana.yml",
   ],
-  "corePluginsDir": "/some/home/dir/core_plugins",
   "homeDir": "/some/home/dir",
   "isDevClusterMaster": false,
   "logDir": "/some/home/dir/log",

src/core/server/config/env.ts

@@ -22,7 +22,7 @@ import process from 'process';
 
 import { pkg } from '../../../utils/package_json';
 
-interface PackageInfo {
+export interface PackageInfo {
   version: string;
   branch: string;
   buildNum: number;
@@ -60,7 +60,6 @@ export class Env {
   }
 
   public readonly configDir: string;
-  public readonly corePluginsDir: string;
   public readonly binDir: string;
   public readonly logDir: string;
   public readonly staticFilesDir: string;
@@ -95,7 +94,6 @@ export class Env {
    */
   constructor(readonly homeDir: string, options: EnvOptions) {
     this.configDir = resolve(this.homeDir, 'config');
-    this.corePluginsDir = resolve(this.homeDir, 'core_plugins');
     this.binDir = resolve(this.homeDir, 'bin');
     this.logDir = resolve(this.homeDir, 'log');
     this.staticFilesDir = resolve(this.homeDir, 'ui');

src/core/server/config/index.ts

@@ -22,5 +22,5 @@ export { RawConfigService } from './raw_config_service';
 export { Config, ConfigPath } from './config';
 /** @internal */
 export { ObjectToConfigAdapter } from './object_to_config_adapter';
-export { Env, CliArgs } from './env';
+export { Env, CliArgs, PackageInfo } from './env';
 export { ConfigWithSchema } from './config_with_schema';

src/core/server/index.test.ts

@@ -22,6 +22,11 @@ jest.mock('./http/http_service', () => ({
   HttpService: jest.fn(() => mockHttpService),
 }));
 
+const mockPluginsService = { start: jest.fn(), stop: jest.fn() };
+jest.mock('./plugins/plugins_service', () => ({
+  PluginsService: jest.fn(() => mockPluginsService),
+}));
+
 const mockLegacyService = { start: jest.fn(), stop: jest.fn() };
 jest.mock('./legacy_compat/legacy_service', () => ({
   LegacyService: jest.fn(() => mockLegacyService),
@@ -45,6 +50,8 @@ afterEach(() => {
   mockConfigService.atPath.mockReset();
   mockHttpService.start.mockReset();
   mockHttpService.stop.mockReset();
+  mockPluginsService.start.mockReset();
+  mockPluginsService.stop.mockReset();
   mockLegacyService.start.mockReset();
   mockLegacyService.stop.mockReset();
 });
@@ -56,11 +63,13 @@ test('starts services on "start"', async () => {
   const server = new Server(mockConfigService as any, logger, env);
 
   expect(mockHttpService.start).not.toHaveBeenCalled();
+  expect(mockPluginsService.start).not.toHaveBeenCalled();
   expect(mockLegacyService.start).not.toHaveBeenCalled();
 
   await server.start();
 
   expect(mockHttpService.start).toHaveBeenCalledTimes(1);
+  expect(mockPluginsService.start).toHaveBeenCalledTimes(1);
   expect(mockLegacyService.start).toHaveBeenCalledTimes(1);
   expect(mockLegacyService.start).toHaveBeenCalledWith(mockHttpServiceStartContract);
 });
@@ -112,10 +121,12 @@ test('stops services on "stop"', async () => {
   await server.start();
 
   expect(mockHttpService.stop).not.toHaveBeenCalled();
+  expect(mockPluginsService.stop).not.toHaveBeenCalled();
   expect(mockLegacyService.stop).not.toHaveBeenCalled();
 
   await server.stop();
 
   expect(mockHttpService.stop).toHaveBeenCalledTimes(1);
+  expect(mockPluginsService.stop).toHaveBeenCalledTimes(1);
   expect(mockLegacyService.stop).toHaveBeenCalledTimes(1);
 });

src/core/server/index.ts

@@ -17,6 +17,8 @@
  * under the License.
  */
 
+import { PluginsModule } from './plugins';
+
 export { bootstrap } from './bootstrap';
 
 import { first } from 'rxjs/operators';
@@ -27,6 +29,7 @@ import { Logger, LoggerFactory } from './logging';
 
 export class Server {
   private readonly http: HttpModule;
+  private readonly plugins: PluginsModule;
   private readonly legacy: LegacyCompatModule;
   private readonly log: Logger;
 
@@ -38,6 +41,7 @@ export class Server {
     this.log = logger.get('server');
 
     this.http = new HttpModule(configService.atPath('server', HttpConfig), logger);
+    this.plugins = new PluginsModule(configService, logger, env);
     this.legacy = new LegacyCompatModule(configService, logger, env);
   }
 
@@ -54,6 +58,7 @@ export class Server {
       httpServerInfo = await this.http.service.start();
     }
 
+    await this.plugins.service.start();
     await this.legacy.service.start(httpServerInfo);
 
     const unhandledConfigPaths = await this.configService.getUnusedPaths();
@@ -70,6 +75,7 @@ export class Server {
     this.log.debug('stopping server');
 
     await this.legacy.service.stop();
+    await this.plugins.service.stop();
     await this.http.service.stop();
   }
 }

src/core/server/plugins/discovery/index.ts

@@ -0,0 +1,21 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { PluginDiscoveryErrorType } from './plugin_discovery_error';
+export { discover } from './plugins_discovery';

src/core/server/plugins/discovery/plugin_discovery.test.ts

@@ -0,0 +1,159 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const mockReaddir = jest.fn();
+const mockReadFile = jest.fn();
+const mockStat = jest.fn();
+jest.mock('fs', () => ({
+  readdir: mockReaddir,
+  readFile: mockReadFile,
+  stat: mockStat,
+}));
+
+import { resolve } from 'path';
+import { map, toArray } from 'rxjs/operators';
+import { logger } from '../../logging/__mocks__';
+import { discover } from './plugins_discovery';
+
+const TEST_PATHS = {
+  scanDirs: {
+    nonEmpty: resolve('scan', 'non-empty'),
+    nonEmpty2: resolve('scan', 'non-empty-2'),
+    nonExistent: resolve('scan', 'non-existent'),
+    empty: resolve('scan', 'empty'),
+  },
+  paths: {
+    existentDir: resolve('path', 'existent-dir'),
+    existentDir2: resolve('path', 'existent-dir-2'),
+    nonDir: resolve('path', 'non-dir'),
+    nonExistent: resolve('path', 'non-existent'),
+  },
+};
+
+beforeEach(() => {
+  mockReaddir.mockImplementation((path, cb) => {
+    if (path === TEST_PATHS.scanDirs.nonEmpty) {
+      cb(null, ['1', '2-no-manifest', '3', '4-incomplete-manifest']);
+    } else if (path === TEST_PATHS.scanDirs.nonEmpty2) {
+      cb(null, ['5-invalid-manifest', '6', '7-non-dir', '8-incompatible-manifest']);
+    } else if (path === TEST_PATHS.scanDirs.nonExistent) {
+      cb(new Error('ENOENT'));
+    } else {
+      cb(null, []);
+    }
+  });
+
+  mockStat.mockImplementation((path, cb) => {
+    if (path.includes('non-existent')) {
+      cb(new Error('ENOENT'));
+    } else {
+      cb(null, { isDirectory: () => !path.includes('non-dir') });
+    }
+  });
+
+  mockReadFile.mockImplementation((path, cb) => {
+    if (path.includes('no-manifest')) {
+      cb(new Error('ENOENT'));
+    } else if (path.includes('invalid-manifest')) {
+      cb(null, Buffer.from('not-json'));
+    } else if (path.includes('incomplete-manifest')) {
+      cb(null, Buffer.from(JSON.stringify({ version: '1' })));
+    } else if (path.includes('incompatible-manifest')) {
+      cb(null, Buffer.from(JSON.stringify({ id: 'plugin', version: '1' })));
+    } else {
+      cb(null, Buffer.from(JSON.stringify({ id: 'plugin', version: '1', kibanaVersion: '1.2.3' })));
+    }
+  });
+});
+
+afterEach(() => {
+  jest.clearAllMocks();
+});
+
+test('properly scans folders and paths', async () => {
+  const { plugin$, error$ } = discover(
+    {
+      initialize: true,
+      scanDirs: Object.values(TEST_PATHS.scanDirs),
+      paths: Object.values(TEST_PATHS.paths),
+    },
+    {
+      branch: 'master',
+      buildNum: 1,
+      buildSha: '',
+      version: '1.2.3',
+    },
+    logger.get()
+  );
+
+  await expect(plugin$.pipe(toArray()).toPromise()).resolves.toEqual(
+    [
+      resolve(TEST_PATHS.scanDirs.nonEmpty, '1'),
+      resolve(TEST_PATHS.scanDirs.nonEmpty, '3'),
+      resolve(TEST_PATHS.scanDirs.nonEmpty2, '6'),
+      resolve(TEST_PATHS.paths.existentDir),
+      resolve(TEST_PATHS.paths.existentDir2),
+    ].map(path => ({
+      manifest: {
+        id: 'plugin',
+        version: '1',
+        kibanaVersion: '1.2.3',
+        optionalPlugins: [],
+        requiredPlugins: [],
+        ui: false,
+      },
+      path,
+    }))
+  );
+
+  await expect(
+    error$
+      .pipe(
+        map(error => error.toString()),
+        toArray()
+      )
+      .toPromise()
+  ).resolves.toEqual([
+    `Error: ENOENT (invalid-scan-dir, ${resolve(TEST_PATHS.scanDirs.nonExistent)})`,
+    `Error: ${resolve(TEST_PATHS.paths.nonDir)} is not a directory. (invalid-plugin-dir, ${resolve(
+      TEST_PATHS.paths.nonDir
+    )})`,
+    `Error: ENOENT (invalid-plugin-dir, ${resolve(TEST_PATHS.paths.nonExistent)})`,
+    `Error: ENOENT (missing-manifest, ${resolve(
+      TEST_PATHS.scanDirs.nonEmpty,
+      '2-no-manifest',
+      'kibana.json'
+    )})`,
+    `Error: Plugin manifest must contain an "id" property. (invalid-manifest, ${resolve(
+      TEST_PATHS.scanDirs.nonEmpty,
+      '4-incomplete-manifest',
+      'kibana.json'
+    )})`,
+    `Error: Unexpected token o in JSON at position 1 (invalid-manifest, ${resolve(
+      TEST_PATHS.scanDirs.nonEmpty2,
+      '5-invalid-manifest',
+      'kibana.json'
+    )})`,
+    `Error: Plugin "plugin" is only compatible with Kibana version "1", but used Kibana version is "1.2.3". (incompatible-version, ${resolve(
+      TEST_PATHS.scanDirs.nonEmpty2,
+      '8-incompatible-manifest',
+      'kibana.json'
+    )})`,
+  ]);
+});