From 18a3cee1cff34e59733a51bd7f9fb2ecd7496e68 Mon Sep 17 00:00:00 2001
From: Rudolf Meijering <skaapgif@gmail.com>
Date: Mon, 3 Feb 2020 18:30:51 +0100
Subject: [PATCH] Perform successful Elasticsearch version check before
 migrations (#51311) (#56629)

* Convert parts of Elasticsearch version check to ts
* Move ES version check to NP
* Improve types
* Wait till for compatible ES nodes before SO migrations
* Don't wait for ES compatibility if skipMigrations=true
* Legacy Elasticsearch plugin integration test
* Make ES compatibility check and migrations logging more visible
* Test for isCompatible=false when ES version check throws
* Start pollEsNodesVersion immediately
* Refactor pollEsNodesVersion
---
 .../elasticsearch/elasticsearch_config.ts     |  14 +-
 .../elasticsearch_service.mock.ts             |   7 +
 .../elasticsearch_service.test.ts             |  11 +-
 .../elasticsearch/elasticsearch_service.ts    |  46 ++-
 src/core/server/elasticsearch/types.ts        |   2 +
 .../version_check/ensure_es_version.test.ts   | 261 ++++++++++++++++++
 .../version_check/ensure_es_version.ts        | 164 +++++++++++
 .../es_kibana_version_compatability.test.ts}  |  18 +-
 .../es_kibana_version_compatability.ts}       |  15 +-
 .../integration_tests/core_services.test.ts   |   6 +-
 .../integration_tests/legacy_service.test.ts  |   2 +-
 .../migrations/kibana/kibana_migrator.test.ts |   6 -
 .../migrations/kibana/kibana_migrator.ts      |  15 +-
 .../saved_objects_service.test.ts             |  61 +++-
 .../saved_objects/saved_objects_service.ts    |  27 +-
 .../core_plugins/elasticsearch/index.d.ts     |   1 +
 .../core_plugins/elasticsearch/index.js       |  14 +-
 .../integration_tests/elasticsearch.test.ts   |  89 ++++++
 .../lib/__tests__/ensure_es_version.js        | 223 ---------------
 .../lib/__tests__/health_check.js             | 151 ----------
 .../elasticsearch/lib/ensure_es_version.js    | 126 ---------
 .../elasticsearch/lib/health_check.js         |  75 -----
 .../elasticsearch/lib/version_health_check.js |  39 +++
 .../lib/version_health_check.test.js          |  71 +++++
 .../default_route_provider.test.ts            |   2 +-
 .../default_route_provider_config.test.ts     |   1 +
 .../max_payload_size.test.js                  |   2 +-
 .../on_post_auth_interceptor.test.ts          |   5 +-
 .../on_request_interceptor.test.ts            |   5 +-
 29 files changed, 806 insertions(+), 653 deletions(-)
 create mode 100644 src/core/server/elasticsearch/version_check/ensure_es_version.test.ts
 create mode 100644 src/core/server/elasticsearch/version_check/ensure_es_version.ts
 rename src/{legacy/core_plugins/elasticsearch/lib/__tests__/is_es_compatible_with_kibana.js => core/server/elasticsearch/version_check/es_kibana_version_compatability.test.ts} (72%)
 rename src/{legacy/core_plugins/elasticsearch/lib/is_es_compatible_with_kibana.js => core/server/elasticsearch/version_check/es_kibana_version_compatability.ts} (76%)
 create mode 100644 src/legacy/core_plugins/elasticsearch/integration_tests/elasticsearch.test.ts
 delete mode 100644 src/legacy/core_plugins/elasticsearch/lib/__tests__/ensure_es_version.js
 delete mode 100644 src/legacy/core_plugins/elasticsearch/lib/__tests__/health_check.js
 delete mode 100644 src/legacy/core_plugins/elasticsearch/lib/ensure_es_version.js
 delete mode 100644 src/legacy/core_plugins/elasticsearch/lib/health_check.js
 create mode 100644 src/legacy/core_plugins/elasticsearch/lib/version_health_check.js
 create mode 100644 src/legacy/core_plugins/elasticsearch/lib/version_health_check.test.js

diff --git a/src/core/server/elasticsearch/elasticsearch_config.ts b/src/core/server/elasticsearch/elasticsearch_config.ts
index 41bbdadab9185..2f4160ad75e9d 100644
--- a/src/core/server/elasticsearch/elasticsearch_config.ts
+++ b/src/core/server/elasticsearch/elasticsearch_config.ts
@@ -103,7 +103,19 @@ const configSchema = schema.object({
   ),
   apiVersion: schema.string({ defaultValue: DEFAULT_API_VERSION }),
   healthCheck: schema.object({ delay: schema.duration({ defaultValue: 2500 }) }),
-  ignoreVersionMismatch: schema.boolean({ defaultValue: false }),
+  ignoreVersionMismatch: schema.conditional(
+    schema.contextRef('dev'),
+    false,
+    schema.boolean({
+      validate: rawValue => {
+        if (rawValue === true) {
+          return '"ignoreVersionMismatch" can only be set to true in development mode';
+        }
+      },
+      defaultValue: false,
+    }),
+    schema.boolean({ defaultValue: false })
+  ),
 });
 
 const deprecations: ConfigDeprecationProvider = () => [
diff --git a/src/core/server/elasticsearch/elasticsearch_service.mock.ts b/src/core/server/elasticsearch/elasticsearch_service.mock.ts
index a4e51ca55b3e7..b8ad375496544 100644
--- a/src/core/server/elasticsearch/elasticsearch_service.mock.ts
+++ b/src/core/server/elasticsearch/elasticsearch_service.mock.ts
@@ -23,6 +23,7 @@ import { IScopedClusterClient } from './scoped_cluster_client';
 import { ElasticsearchConfig } from './elasticsearch_config';
 import { ElasticsearchService } from './elasticsearch_service';
 import { InternalElasticsearchServiceSetup, ElasticsearchServiceSetup } from './types';
+import { NodesVersionCompatibility } from './version_check/ensure_es_version';
 
 const createScopedClusterClientMock = (): jest.Mocked<IScopedClusterClient> => ({
   callAsInternalUser: jest.fn(),
@@ -71,6 +72,12 @@ type MockedInternalElasticSearchServiceSetup = jest.Mocked<
 const createInternalSetupContractMock = () => {
   const setupContract: MockedInternalElasticSearchServiceSetup = {
     ...createSetupContractMock(),
+    esNodesCompatibility$: new BehaviorSubject<NodesVersionCompatibility>({
+      isCompatible: true,
+      incompatibleNodes: [],
+      warningNodes: [],
+      kibanaVersion: '8.0.0',
+    }),
     legacy: {
       config$: new BehaviorSubject({} as ElasticsearchConfig),
     },
diff --git a/src/core/server/elasticsearch/elasticsearch_service.test.ts b/src/core/server/elasticsearch/elasticsearch_service.test.ts
index 5a7d223fec7ad..022a03e01d37d 100644
--- a/src/core/server/elasticsearch/elasticsearch_service.test.ts
+++ b/src/core/server/elasticsearch/elasticsearch_service.test.ts
@@ -31,6 +31,7 @@ import { httpServiceMock } from '../http/http_service.mock';
 import { ElasticsearchConfig } from './elasticsearch_config';
 import { ElasticsearchService } from './elasticsearch_service';
 import { elasticsearchServiceMock } from './elasticsearch_service.mock';
+import { duration } from 'moment';
 
 let elasticsearchService: ElasticsearchService;
 const configService = configServiceMock.create();
@@ -41,7 +42,7 @@ configService.atPath.mockReturnValue(
   new BehaviorSubject({
     hosts: ['http://1.2.3.4'],
     healthCheck: {
-      delay: 2000,
+      delay: duration(2000),
     },
     ssl: {
       verificationMode: 'none',
@@ -125,7 +126,7 @@ describe('#setup', () => {
       const config = MockClusterClient.mock.calls[0][0];
       expect(config).toMatchInlineSnapshot(`
 Object {
-  "healthCheckDelay": 2000,
+  "healthCheckDelay": "PT2S",
   "hosts": Array [
     "http://8.8.8.8",
   ],
@@ -150,7 +151,7 @@ Object {
       const config = MockClusterClient.mock.calls[0][0];
       expect(config).toMatchInlineSnapshot(`
 Object {
-  "healthCheckDelay": 2000,
+  "healthCheckDelay": "PT2S",
   "hosts": Array [
     "http://1.2.3.4",
   ],
@@ -174,7 +175,7 @@ Object {
         new BehaviorSubject({
           hosts: ['http://1.2.3.4', 'http://9.8.7.6'],
           healthCheck: {
-            delay: 2000,
+            delay: duration(2000),
           },
           ssl: {
             verificationMode: 'none',
@@ -196,7 +197,7 @@ Object {
       const config = MockClusterClient.mock.calls[0][0];
       expect(config).toMatchInlineSnapshot(`
         Object {
-          "healthCheckDelay": 2000,
+          "healthCheckDelay": "PT2S",
           "hosts": Array [
             "http://8.8.8.8",
           ],
diff --git a/src/core/server/elasticsearch/elasticsearch_service.ts b/src/core/server/elasticsearch/elasticsearch_service.ts
index aba246ce66fb5..4d897dbd8f7aa 100644
--- a/src/core/server/elasticsearch/elasticsearch_service.ts
+++ b/src/core/server/elasticsearch/elasticsearch_service.ts
@@ -30,6 +30,7 @@ import { ElasticsearchConfig, ElasticsearchConfigType } from './elasticsearch_co
 import { InternalHttpServiceSetup, GetAuthHeaders } from '../http/';
 import { InternalElasticsearchServiceSetup } from './types';
 import { CallAPIOptions } from './api_types';
+import { pollEsNodesVersion } from './version_check/ensure_es_version';
 
 /** @internal */
 interface CoreClusterClients {
@@ -46,9 +47,17 @@ interface SetupDeps {
 export class ElasticsearchService implements CoreService<InternalElasticsearchServiceSetup> {
   private readonly log: Logger;
   private readonly config$: Observable<ElasticsearchConfig>;
-  private subscription?: Subscription;
+  private subscriptions: {
+    client?: Subscription;
+    esNodesCompatibility?: Subscription;
+  } = {
+    client: undefined,
+    esNodesCompatibility: undefined,
+  };
+  private kibanaVersion: string;
 
   constructor(private readonly coreContext: CoreContext) {
+    this.kibanaVersion = coreContext.env.packageInfo.version;
     this.log = coreContext.logger.get('elasticsearch-service');
     this.config$ = coreContext.configService
       .atPath<ElasticsearchConfigType>('elasticsearch')
@@ -60,7 +69,7 @@ export class ElasticsearchService implements CoreService<InternalElasticsearchSe
 
     const clients$ = this.config$.pipe(
       filter(() => {
-        if (this.subscription !== undefined) {
+        if (this.subscriptions.client !== undefined) {
           this.log.error('Clients cannot be changed after they are created');
           return false;
         }
@@ -91,7 +100,7 @@ export class ElasticsearchService implements CoreService<InternalElasticsearchSe
       publishReplay(1)
     ) as ConnectableObservable<CoreClusterClients>;
 
-    this.subscription = clients$.connect();
+    this.subscriptions.client = clients$.connect();
 
     const config = await this.config$.pipe(first()).toPromise();
 
@@ -149,11 +158,31 @@ export class ElasticsearchService implements CoreService<InternalElasticsearchSe
       },
     };
 
+    const esNodesCompatibility$ = pollEsNodesVersion({
+      callWithInternalUser: adminClient.callAsInternalUser,
+      log: this.log,
+      ignoreVersionMismatch: config.ignoreVersionMismatch,
+      esVersionCheckInterval: config.healthCheckDelay.asMilliseconds(),
+      kibanaVersion: this.kibanaVersion,
+    }).pipe(publishReplay(1));
+
+    this.subscriptions.esNodesCompatibility = (esNodesCompatibility$ as ConnectableObservable<
+      unknown
+    >).connect();
+
+    // TODO: Move to Status Service https://github.com/elastic/kibana/issues/41983
+    esNodesCompatibility$.subscribe(({ isCompatible, message }) => {
+      if (!isCompatible && message) {
+        this.log.error(message);
+      }
+    });
+
     return {
       legacy: { config$: clients$.pipe(map(clients => clients.config)) },
 
       adminClient,
       dataClient,
+      esNodesCompatibility$,
 
       createClient: (type: string, clientConfig: Partial<ElasticsearchClientConfig> = {}) => {
         const finalConfig = merge({}, config, clientConfig);
@@ -166,11 +195,12 @@ export class ElasticsearchService implements CoreService<InternalElasticsearchSe
 
   public async stop() {
     this.log.debug('Stopping elasticsearch service');
-
-    if (this.subscription !== undefined) {
-      this.subscription.unsubscribe();
-      this.subscription = undefined;
-    }
+    // TODO(TS-3.7-ESLINT)
+    // eslint-disable-next-line no-unused-expressions
+    this.subscriptions.client?.unsubscribe();
+    // eslint-disable-next-line no-unused-expressions
+    this.subscriptions.esNodesCompatibility?.unsubscribe();
+    this.subscriptions = { client: undefined, esNodesCompatibility: undefined };
   }
 
   private createClusterClient(
diff --git a/src/core/server/elasticsearch/types.ts b/src/core/server/elasticsearch/types.ts
index 899b273c5c60a..90cfdcc035d8e 100644
--- a/src/core/server/elasticsearch/types.ts
+++ b/src/core/server/elasticsearch/types.ts
@@ -21,6 +21,7 @@ import { Observable } from 'rxjs';
 import { ElasticsearchConfig } from './elasticsearch_config';
 import { ElasticsearchClientConfig } from './elasticsearch_client_config';
 import { IClusterClient, ICustomClusterClient } from './cluster_client';
+import { NodesVersionCompatibility } from './version_check/ensure_es_version';
 
 /**
  * @public
@@ -77,4 +78,5 @@ export interface InternalElasticsearchServiceSetup extends ElasticsearchServiceS
   readonly legacy: {
     readonly config$: Observable<ElasticsearchConfig>;
   };
+  esNodesCompatibility$: Observable<NodesVersionCompatibility>;
 }
diff --git a/src/core/server/elasticsearch/version_check/ensure_es_version.test.ts b/src/core/server/elasticsearch/version_check/ensure_es_version.test.ts
new file mode 100644
index 0000000000000..4989c4a31295c
--- /dev/null
+++ b/src/core/server/elasticsearch/version_check/ensure_es_version.test.ts
@@ -0,0 +1,261 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { mapNodesVersionCompatibility, pollEsNodesVersion, NodesInfo } from './ensure_es_version';
+import { loggingServiceMock } from '../../logging/logging_service.mock';
+import { take, delay } from 'rxjs/operators';
+import { TestScheduler } from 'rxjs/testing';
+import { of } from 'rxjs';
+
+const mockLoggerFactory = loggingServiceMock.create();
+const mockLogger = mockLoggerFactory.get('mock logger');
+
+const KIBANA_VERSION = '5.1.0';
+
+function createNodes(...versions: string[]): NodesInfo {
+  const nodes = {} as any;
+  versions
+    .map(version => {
+      return {
+        version,
+        http: {
+          publish_address: 'http_address',
+        },
+        ip: 'ip',
+      };
+    })
+    .forEach((node, i) => {
+      nodes[`node-${i}`] = node;
+    });
+
+  return { nodes };
+}
+
+describe('mapNodesVersionCompatibility', () => {
+  function createNodesInfoWithoutHTTP(version: string): NodesInfo {
+    return { nodes: { 'node-without-http': { version, ip: 'ip' } } } as any;
+  }
+
+  it('returns isCompatible=true with a single node that matches', async () => {
+    const nodesInfo = createNodes('5.1.0');
+    const result = await mapNodesVersionCompatibility(nodesInfo, KIBANA_VERSION, false);
+    expect(result.isCompatible).toBe(true);
+  });
+
+  it('returns isCompatible=true with multiple nodes that satisfy', async () => {
+    const nodesInfo = createNodes('5.1.0', '5.2.0', '5.1.1-Beta1');
+    const result = await mapNodesVersionCompatibility(nodesInfo, KIBANA_VERSION, false);
+    expect(result.isCompatible).toBe(true);
+  });
+
+  it('returns isCompatible=false for a single node that is out of date', () => {
+    // 5.0.0 ES is too old to work with a 5.1.0 version of Kibana.
+    const nodesInfo = createNodes('5.1.0', '5.2.0', '5.0.0');
+    const result = mapNodesVersionCompatibility(nodesInfo, KIBANA_VERSION, false);
+    expect(result.isCompatible).toBe(false);
+    expect(result.message).toMatchInlineSnapshot(
+      `"This version of Kibana (v5.1.0) is incompatible with the following Elasticsearch nodes in your cluster: v5.0.0 @ http_address (ip)"`
+    );
+  });
+
+  it('returns isCompatible=false for an incompatible node without http publish address', async () => {
+    const nodesInfo = createNodesInfoWithoutHTTP('6.1.1');
+    const result = mapNodesVersionCompatibility(nodesInfo, KIBANA_VERSION, false);
+    expect(result.isCompatible).toBe(false);
+    expect(result.message).toMatchInlineSnapshot(
+      `"This version of Kibana (v5.1.0) is incompatible with the following Elasticsearch nodes in your cluster: v6.1.1 @ undefined (ip)"`
+    );
+  });
+
+  it('returns isCompatible=true for outdated nodes when ignoreVersionMismatch=true', async () => {
+    // 5.0.0 ES is too old to work with a 5.1.0 version of Kibana.
+    const nodesInfo = createNodes('5.1.0', '5.2.0', '5.0.0');
+    const ignoreVersionMismatch = true;
+    const result = mapNodesVersionCompatibility(nodesInfo, KIBANA_VERSION, ignoreVersionMismatch);
+    expect(result.isCompatible).toBe(true);
+    expect(result.message).toMatchInlineSnapshot(
+      `"Ignoring version incompatibility between Kibana v5.1.0 and the following Elasticsearch nodes: v5.0.0 @ http_address (ip)"`
+    );
+  });
+
+  it('returns isCompatible=true with a message if a node is only off by a patch version', () => {
+    const result = mapNodesVersionCompatibility(createNodes('5.1.1'), KIBANA_VERSION, false);
+    expect(result.isCompatible).toBe(true);
+    expect(result.message).toMatchInlineSnapshot(
+      `"You're running Kibana 5.1.0 with some different versions of Elasticsearch. Update Kibana or Elasticsearch to the same version to prevent compatibility issues: v5.1.1 @ http_address (ip)"`
+    );
+  });
+
+  it('returns isCompatible=true with a message if a node is only off by a patch version and without http publish address', async () => {
+    const result = mapNodesVersionCompatibility(createNodes('5.1.1'), KIBANA_VERSION, false);
+    expect(result.isCompatible).toBe(true);
+    expect(result.message).toMatchInlineSnapshot(
+      `"You're running Kibana 5.1.0 with some different versions of Elasticsearch. Update Kibana or Elasticsearch to the same version to prevent compatibility issues: v5.1.1 @ http_address (ip)"`
+    );
+  });
+});
+
+describe('pollEsNodesVersion', () => {
+  const callWithInternalUser = jest.fn();
+  const getTestScheduler = () =>
+    new TestScheduler((actual, expected) => {
+      expect(actual).toEqual(expected);
+    });
+
+  beforeEach(() => {
+    callWithInternalUser.mockClear();
+  });
+
+  it('returns iscCompatible=false and keeps polling when a poll request throws', done => {
+    expect.assertions(3);
+    const expectedCompatibilityResults = [false, false, true];
+    jest.clearAllMocks();
+    callWithInternalUser.mockResolvedValueOnce(createNodes('5.1.0', '5.2.0', '5.0.0'));
+    callWithInternalUser.mockRejectedValueOnce(new Error('mock request error'));
+    callWithInternalUser.mockResolvedValueOnce(createNodes('5.1.0', '5.2.0', '5.1.1-Beta1'));
+    pollEsNodesVersion({
+      callWithInternalUser,
+      esVersionCheckInterval: 1,
+      ignoreVersionMismatch: false,
+      kibanaVersion: KIBANA_VERSION,
+      log: mockLogger,
+    })
+      .pipe(take(3))
+      .subscribe({
+        next: result => {
+          expect(result.isCompatible).toBe(expectedCompatibilityResults.shift());
+        },
+        complete: done,
+        error: done,
+      });
+  });
+
+  it('returns compatibility results', done => {
+    expect.assertions(1);
+    const nodes = createNodes('5.1.0', '5.2.0', '5.0.0');
+    callWithInternalUser.mockResolvedValueOnce(nodes);
+    pollEsNodesVersion({
+      callWithInternalUser,
+      esVersionCheckInterval: 1,
+      ignoreVersionMismatch: false,
+      kibanaVersion: KIBANA_VERSION,
+      log: mockLogger,
+    })
+      .pipe(take(1))
+      .subscribe({
+        next: result => {
+          expect(result).toEqual(mapNodesVersionCompatibility(nodes, KIBANA_VERSION, false));
+        },
+        complete: done,
+        error: done,
+      });
+  });
+
+  it('only emits if the node versions changed since the previous poll', done => {
+    expect.assertions(4);
+    callWithInternalUser.mockResolvedValueOnce(createNodes('5.1.0', '5.2.0', '5.0.0')); // emit
+    callWithInternalUser.mockResolvedValueOnce(createNodes('5.0.0', '5.1.0', '5.2.0')); // ignore, same versions, different ordering
+    callWithInternalUser.mockResolvedValueOnce(createNodes('5.1.1', '5.2.0', '5.0.0')); // emit
+    callWithInternalUser.mockResolvedValueOnce(createNodes('5.1.1', '5.1.2', '5.1.3')); // emit
+    callWithInternalUser.mockResolvedValueOnce(createNodes('5.1.1', '5.1.2', '5.1.3')); // ignore
+    callWithInternalUser.mockResolvedValueOnce(createNodes('5.0.0', '5.1.0', '5.2.0')); // emit, different from previous version
+
+    pollEsNodesVersion({
+      callWithInternalUser,
+      esVersionCheckInterval: 1,
+      ignoreVersionMismatch: false,
+      kibanaVersion: KIBANA_VERSION,
+      log: mockLogger,
+    })
+      .pipe(take(4))
+      .subscribe({
+        next: result => expect(result).toBeDefined(),
+        complete: done,
+        error: done,
+      });
+  });
+
+  it('starts polling immediately and then every esVersionCheckInterval', () => {
+    expect.assertions(1);
+    callWithInternalUser.mockReturnValueOnce([createNodes('5.1.0', '5.2.0', '5.0.0')]);
+    callWithInternalUser.mockReturnValueOnce([createNodes('5.1.1', '5.2.0', '5.0.0')]);
+
+    getTestScheduler().run(({ expectObservable }) => {
+      const expected = 'a 99ms (b|)';
+
+      const esNodesCompatibility$ = pollEsNodesVersion({
+        callWithInternalUser,
+        esVersionCheckInterval: 100,
+        ignoreVersionMismatch: false,
+        kibanaVersion: KIBANA_VERSION,
+        log: mockLogger,
+      }).pipe(take(2));
+
+      expectObservable(esNodesCompatibility$).toBe(expected, {
+        a: mapNodesVersionCompatibility(
+          createNodes('5.1.0', '5.2.0', '5.0.0'),
+          KIBANA_VERSION,
+          false
+        ),
+        b: mapNodesVersionCompatibility(
+          createNodes('5.1.1', '5.2.0', '5.0.0'),
+          KIBANA_VERSION,
+          false
+        ),
+      });
+    });
+  });
+
+  it('waits for es version check requests to complete before scheduling the next one', () => {
+    expect.assertions(2);
+
+    getTestScheduler().run(({ expectObservable }) => {
+      const expected = '100ms a 99ms (b|)';
+
+      callWithInternalUser.mockReturnValueOnce(
+        of(createNodes('5.1.0', '5.2.0', '5.0.0')).pipe(delay(100))
+      );
+      callWithInternalUser.mockReturnValueOnce(
+        of(createNodes('5.1.1', '5.2.0', '5.0.0')).pipe(delay(100))
+      );
+
+      const esNodesCompatibility$ = pollEsNodesVersion({
+        callWithInternalUser,
+        esVersionCheckInterval: 10,
+        ignoreVersionMismatch: false,
+        kibanaVersion: KIBANA_VERSION,
+        log: mockLogger,
+      }).pipe(take(2));
+
+      expectObservable(esNodesCompatibility$).toBe(expected, {
+        a: mapNodesVersionCompatibility(
+          createNodes('5.1.0', '5.2.0', '5.0.0'),
+          KIBANA_VERSION,
+          false
+        ),
+        b: mapNodesVersionCompatibility(
+          createNodes('5.1.1', '5.2.0', '5.0.0'),
+          KIBANA_VERSION,
+          false
+        ),
+      });
+    });
+
+    expect(callWithInternalUser).toHaveBeenCalledTimes(2);
+  });
+});
diff --git a/src/core/server/elasticsearch/version_check/ensure_es_version.ts b/src/core/server/elasticsearch/version_check/ensure_es_version.ts
new file mode 100644
index 0000000000000..3e760ec0efabd
--- /dev/null
+++ b/src/core/server/elasticsearch/version_check/ensure_es_version.ts
@@ -0,0 +1,164 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * ES and Kibana versions are locked, so Kibana should require that ES has the same version as
+ * that defined in Kibana's package.json.
+ */
+
+import { timer, of, from, Observable } from 'rxjs';
+import { map, distinctUntilChanged, catchError, exhaustMap } from 'rxjs/operators';
+import {
+  esVersionCompatibleWithKibana,
+  esVersionEqualsKibana,
+} from './es_kibana_version_compatability';
+import { Logger } from '../../logging';
+import { APICaller } from '..';
+
+export interface PollEsNodesVersionOptions {
+  callWithInternalUser: APICaller;
+  log: Logger;
+  kibanaVersion: string;
+  ignoreVersionMismatch: boolean;
+  esVersionCheckInterval: number;
+}
+
+interface NodeInfo {
+  version: string;
+  ip: string;
+  http: {
+    publish_address: string;
+  };
+  name: string;
+}
+
+export interface NodesInfo {
+  nodes: {
+    [key: string]: NodeInfo;
+  };
+}
+
+export interface NodesVersionCompatibility {
+  isCompatible: boolean;
+  message?: string;
+  incompatibleNodes: NodeInfo[];
+  warningNodes: NodeInfo[];
+  kibanaVersion: string;
+}
+
+function getHumanizedNodeName(node: NodeInfo) {
+  const publishAddress = node?.http?.publish_address + ' ' || '';
+  return 'v' + node.version + ' @ ' + publishAddress + '(' + node.ip + ')';
+}
+
+export function mapNodesVersionCompatibility(
+  nodesInfo: NodesInfo,
+  kibanaVersion: string,
+  ignoreVersionMismatch: boolean
+): NodesVersionCompatibility {
+  if (Object.keys(nodesInfo.nodes).length === 0) {
+    return {
+      isCompatible: false,
+      message: 'Unable to retrieve version information from Elasticsearch nodes.',
+      incompatibleNodes: [],
+      warningNodes: [],
+      kibanaVersion,
+    };
+  }
+  const nodes = Object.keys(nodesInfo.nodes)
+    .sort() // Sorting ensures a stable node ordering for comparison
+    .map(key => nodesInfo.nodes[key])
+    .map(node => Object.assign({}, node, { name: getHumanizedNodeName(node) }));
+
+  // Aggregate incompatible ES nodes.
+  const incompatibleNodes = nodes.filter(
+    node => !esVersionCompatibleWithKibana(node.version, kibanaVersion)
+  );
+
+  // Aggregate ES nodes which should prompt a Kibana upgrade. It's acceptable
+  // if ES and Kibana versions are not the same as long as they are not
+  // incompatible, but we should warn about it.
+  // Ignore version qualifiers https://github.com/elastic/elasticsearch/issues/36859
+  const warningNodes = nodes.filter(node => !esVersionEqualsKibana(node.version, kibanaVersion));
+
+  // Note: If incompatible and warning nodes are present `message` only contains
+  // an incompatibility notice.
+  let message;
+  if (incompatibleNodes.length > 0) {
+    const incompatibleNodeNames = incompatibleNodes.map(node => node.name).join(', ');
+    if (ignoreVersionMismatch) {
+      message = `Ignoring version incompatibility between Kibana v${kibanaVersion} and the following Elasticsearch nodes: ${incompatibleNodeNames}`;
+    } else {
+      message = `This version of Kibana (v${kibanaVersion}) is incompatible with the following Elasticsearch nodes in your cluster: ${incompatibleNodeNames}`;
+    }
+  } else if (warningNodes.length > 0) {
+    const warningNodeNames = warningNodes.map(node => node.name).join(', ');
+    message =
+      `You're running Kibana ${kibanaVersion} with some different versions of ` +
+      'Elasticsearch. Update Kibana or Elasticsearch to the same ' +
+      `version to prevent compatibility issues: ${warningNodeNames}`;
+  }
+
+  return {
+    isCompatible: ignoreVersionMismatch || incompatibleNodes.length === 0,
+    message,
+    incompatibleNodes,
+    warningNodes,
+    kibanaVersion,
+  };
+}
+
+// Returns true if two NodesVersionCompatibility entries match
+function compareNodes(prev: NodesVersionCompatibility, curr: NodesVersionCompatibility) {
+  const nodesEqual = (n: NodeInfo, m: NodeInfo) => n.ip === m.ip && n.version === m.version;
+  return (
+    curr.isCompatible === prev.isCompatible &&
+    curr.incompatibleNodes.length === prev.incompatibleNodes.length &&
+    curr.warningNodes.length === prev.warningNodes.length &&
+    curr.incompatibleNodes.every((node, i) => nodesEqual(node, prev.incompatibleNodes[i])) &&
+    curr.warningNodes.every((node, i) => nodesEqual(node, prev.warningNodes[i]))
+  );
+}
+
+export const pollEsNodesVersion = ({
+  callWithInternalUser,
+  log,
+  kibanaVersion,
+  ignoreVersionMismatch,
+  esVersionCheckInterval: healthCheckInterval,
+}: PollEsNodesVersionOptions): Observable<any> => {
+  log.debug('Checking Elasticsearch version');
+  return timer(0, healthCheckInterval).pipe(
+    exhaustMap(() => {
+      return from(
+        callWithInternalUser('nodes.info', {
+          filterPath: ['nodes.*.version', 'nodes.*.http.publish_address', 'nodes.*.ip'],
+        })
+      ).pipe(
+        catchError(_err => {
+          return of({ nodes: {} });
+        })
+      );
+    }),
+    map((nodesInfo: NodesInfo) =>
+      mapNodesVersionCompatibility(nodesInfo, kibanaVersion, ignoreVersionMismatch)
+    ),
+    distinctUntilChanged(compareNodes) // Only emit if there are new nodes or versions
+  );
+};
diff --git a/src/legacy/core_plugins/elasticsearch/lib/__tests__/is_es_compatible_with_kibana.js b/src/core/server/elasticsearch/version_check/es_kibana_version_compatability.test.ts
similarity index 72%
rename from src/legacy/core_plugins/elasticsearch/lib/__tests__/is_es_compatible_with_kibana.js
rename to src/core/server/elasticsearch/version_check/es_kibana_version_compatability.test.ts
index 092c0ecf1071c..152f25c813881 100644
--- a/src/legacy/core_plugins/elasticsearch/lib/__tests__/is_es_compatible_with_kibana.js
+++ b/src/core/server/elasticsearch/version_check/es_kibana_version_compatability.test.ts
@@ -17,41 +17,39 @@
  * under the License.
  */
 
-import expect from '@kbn/expect';
-
-import isEsCompatibleWithKibana from '../is_es_compatible_with_kibana';
+import { esVersionCompatibleWithKibana } from './es_kibana_version_compatability';
 
 describe('plugins/elasticsearch', () => {
   describe('lib/is_es_compatible_with_kibana', () => {
     describe('returns false', () => {
       it('when ES major is greater than Kibana major', () => {
-        expect(isEsCompatibleWithKibana('1.0.0', '0.0.0')).to.be(false);
+        expect(esVersionCompatibleWithKibana('1.0.0', '0.0.0')).toBe(false);
       });
 
       it('when ES major is less than Kibana major', () => {
-        expect(isEsCompatibleWithKibana('0.0.0', '1.0.0')).to.be(false);
+        expect(esVersionCompatibleWithKibana('0.0.0', '1.0.0')).toBe(false);
       });
 
       it('when majors are equal, but ES minor is less than Kibana minor', () => {
-        expect(isEsCompatibleWithKibana('1.0.0', '1.1.0')).to.be(false);
+        expect(esVersionCompatibleWithKibana('1.0.0', '1.1.0')).toBe(false);
       });
     });
 
     describe('returns true', () => {
       it('when version numbers are the same', () => {
-        expect(isEsCompatibleWithKibana('1.1.1', '1.1.1')).to.be(true);
+        expect(esVersionCompatibleWithKibana('1.1.1', '1.1.1')).toBe(true);
       });
 
       it('when majors are equal, and ES minor is greater than Kibana minor', () => {
-        expect(isEsCompatibleWithKibana('1.1.0', '1.0.0')).to.be(true);
+        expect(esVersionCompatibleWithKibana('1.1.0', '1.0.0')).toBe(true);
       });
 
       it('when majors and minors are equal, and ES patch is greater than Kibana patch', () => {
-        expect(isEsCompatibleWithKibana('1.1.1', '1.1.0')).to.be(true);
+        expect(esVersionCompatibleWithKibana('1.1.1', '1.1.0')).toBe(true);
       });
 
       it('when majors and minors are equal, but ES patch is less than Kibana patch', () => {
-        expect(isEsCompatibleWithKibana('1.1.0', '1.1.1')).to.be(true);
+        expect(esVersionCompatibleWithKibana('1.1.0', '1.1.1')).toBe(true);
       });
     });
   });
diff --git a/src/legacy/core_plugins/elasticsearch/lib/is_es_compatible_with_kibana.js b/src/core/server/elasticsearch/version_check/es_kibana_version_compatability.ts
similarity index 76%
rename from src/legacy/core_plugins/elasticsearch/lib/is_es_compatible_with_kibana.js
rename to src/core/server/elasticsearch/version_check/es_kibana_version_compatability.ts
index 4afbd488d2946..28b9c0a23e672 100644
--- a/src/legacy/core_plugins/elasticsearch/lib/is_es_compatible_with_kibana.js
+++ b/src/core/server/elasticsearch/version_check/es_kibana_version_compatability.ts
@@ -17,15 +17,14 @@
  * under the License.
  */
 
+import semver, { coerce } from 'semver';
+
 /**
- * Let's weed out the ES versions that won't work with a given Kibana version.
+ * Checks for the compatibilitiy between Elasticsearch and Kibana versions
  * 1. Major version differences will never work together.
  * 2. Older versions of ES won't work with newer versions of Kibana.
  */
-
-import semver from 'semver';
-
-export default function isEsCompatibleWithKibana(esVersion, kibanaVersion) {
+export function esVersionCompatibleWithKibana(esVersion: string, kibanaVersion: string) {
   const esVersionNumbers = {
     major: semver.major(esVersion),
     minor: semver.minor(esVersion),
@@ -50,3 +49,9 @@ export default function isEsCompatibleWithKibana(esVersion, kibanaVersion) {
 
   return true;
 }
+
+export function esVersionEqualsKibana(nodeVersion: string, kibanaVersion: string) {
+  const nodeSemVer = coerce(nodeVersion);
+  const kibanaSemver = coerce(kibanaVersion);
+  return nodeSemVer && kibanaSemver && nodeSemVer.version === kibanaSemver.version;
+}
diff --git a/src/core/server/http/integration_tests/core_services.test.ts b/src/core/server/http/integration_tests/core_services.test.ts
index 65c4f1432721d..0d4570a34ca3c 100644
--- a/src/core/server/http/integration_tests/core_services.test.ts
+++ b/src/core/server/http/integration_tests/core_services.test.ts
@@ -46,7 +46,7 @@ describe('http service', () => {
 
       let root: ReturnType<typeof kbnTestServer.createRoot>;
       beforeEach(async () => {
-        root = kbnTestServer.createRoot();
+        root = kbnTestServer.createRoot({ migrations: { skip: true } });
       }, 30000);
 
       afterEach(async () => {
@@ -180,7 +180,7 @@ describe('http service', () => {
     describe('#basePath()', () => {
       let root: ReturnType<typeof kbnTestServer.createRoot>;
       beforeEach(async () => {
-        root = kbnTestServer.createRoot();
+        root = kbnTestServer.createRoot({ migrations: { skip: true } });
       }, 30000);
 
       afterEach(async () => await root.shutdown());
@@ -209,7 +209,7 @@ describe('http service', () => {
   describe('elasticsearch', () => {
     let root: ReturnType<typeof kbnTestServer.createRoot>;
     beforeEach(async () => {
-      root = kbnTestServer.createRoot();
+      root = kbnTestServer.createRoot({ migrations: { skip: true } });
     }, 30000);
 
     afterEach(async () => {
diff --git a/src/core/server/legacy/integration_tests/legacy_service.test.ts b/src/core/server/legacy/integration_tests/legacy_service.test.ts
index da2550f2ae799..e8bcf7a42d192 100644
--- a/src/core/server/legacy/integration_tests/legacy_service.test.ts
+++ b/src/core/server/legacy/integration_tests/legacy_service.test.ts
@@ -22,7 +22,7 @@ describe('legacy service', () => {
   describe('http server', () => {
     let root: ReturnType<typeof kbnTestServer.createRoot>;
     beforeEach(() => {
-      root = kbnTestServer.createRoot();
+      root = kbnTestServer.createRoot({ migrations: { skip: true } });
     }, 30000);
 
     afterEach(async () => await root.shutdown());
diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts
index b89abc596ad18..c6a72eb53d6c4 100644
--- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts
+++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts
@@ -59,12 +59,6 @@ describe('KibanaMigrator', () => {
   });
 
   describe('runMigrations', () => {
-    it('resolves isMigrated if migrations were skipped', async () => {
-      const skipMigrations = true;
-      const result = await new KibanaMigrator(mockOptions()).runMigrations(skipMigrations);
-      expect(result).toEqual([{ status: 'skipped' }, { status: 'skipped' }]);
-    });
-
     it('only runs migrations once if called multiple times', async () => {
       const options = mockOptions();
       const clusterStub = jest.fn<any, any>(() => ({ status: 404 }));
diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts
index c35e8dd90b5b1..747b48a540109 100644
--- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts
+++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts
@@ -107,24 +107,15 @@ export class KibanaMigrator {
    *    The promise resolves with an array of migration statuses, one for each
    *    elasticsearch index which was migrated.
    */
-  public runMigrations(skipMigrations: boolean = false): Promise<Array<{ status: string }>> {
+  public runMigrations(): Promise<Array<{ status: string }>> {
     if (this.migrationResult === undefined) {
-      this.migrationResult = this.runMigrationsInternal(skipMigrations);
+      this.migrationResult = this.runMigrationsInternal();
     }
 
     return this.migrationResult;
   }
 
-  private runMigrationsInternal(skipMigrations: boolean) {
-    if (skipMigrations) {
-      this.log.warn(
-        'Skipping Saved Object migrations on startup. Note: Individual documents will still be migrated when read or written.'
-      );
-      return Promise.resolve(
-        Object.keys(this.mappingProperties).map(() => ({ status: 'skipped' }))
-      );
-    }
-
+  private runMigrationsInternal() {
     const kibanaIndexName = this.kibanaConfig.index;
     const indexMap = createIndexMap({
       config: this.config,
diff --git a/src/core/server/saved_objects/saved_objects_service.test.ts b/src/core/server/saved_objects/saved_objects_service.test.ts
index deb6f98e17b7b..2bd1335e4d031 100644
--- a/src/core/server/saved_objects/saved_objects_service.test.ts
+++ b/src/core/server/saved_objects/saved_objects_service.test.ts
@@ -26,12 +26,24 @@ import { KibanaMigrator, mockKibanaMigratorInstance } from './migrations/kibana/
 import * as legacyElasticsearch from 'elasticsearch';
 import { Env } from '../config';
 import { configServiceMock } from '../mocks';
-
-afterEach(() => {
-  jest.clearAllMocks();
-});
+import { elasticsearchServiceMock } from '../elasticsearch/elasticsearch_service.mock';
+import { legacyServiceMock } from '../legacy/legacy_service.mock';
+import { BehaviorSubject } from 'rxjs';
+import { NodesVersionCompatibility } from '../elasticsearch/version_check/ensure_es_version';
 
 describe('SavedObjectsService', () => {
+  const createSetupDeps = () => {
+    const elasticsearchMock = elasticsearchServiceMock.createInternalSetup();
+    return {
+      elasticsearch: elasticsearchMock,
+      legacyPlugins: legacyServiceMock.createDiscoverPlugins(),
+    };
+  };
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
   describe('#setup()', () => {
     it('creates a KibanaMigrator which retries NoConnections errors from callAsInternalUser', async () => {
       const coreContext = mockCoreContext.create();
@@ -73,7 +85,7 @@ describe('SavedObjectsService', () => {
 
       await soService.setup(coreSetup);
       await soService.start({});
-      expect(mockKibanaMigratorInstance.runMigrations).toHaveBeenCalledWith(true);
+      expect(mockKibanaMigratorInstance.runMigrations).not.toHaveBeenCalled();
     });
 
     it('skips KibanaMigrator migrations when migrations.skip=true', async () => {
@@ -87,24 +99,51 @@ describe('SavedObjectsService', () => {
 
       await soService.setup(coreSetup);
       await soService.start({});
-      expect(mockKibanaMigratorInstance.runMigrations).toHaveBeenCalledWith(true);
+      expect(mockKibanaMigratorInstance.runMigrations).not.toHaveBeenCalled();
+    });
+
+    it('waits for all es nodes to be compatible before running migrations', async done => {
+      expect.assertions(2);
+      const configService = configServiceMock.create({ atPath: { skip: false } });
+      const coreContext = mockCoreContext.create({ configService });
+      const soService = new SavedObjectsService(coreContext);
+      const setupDeps = createSetupDeps();
+      // Create an new subject so that we can control when isCompatible=true
+      // is emitted.
+      setupDeps.elasticsearch.esNodesCompatibility$ = new BehaviorSubject({
+        isCompatible: false,
+        incompatibleNodes: [],
+        warningNodes: [],
+        kibanaVersion: '8.0.0',
+      });
+      await soService.setup(setupDeps);
+      soService.start({});
+      expect(mockKibanaMigratorInstance.runMigrations).toHaveBeenCalledTimes(0);
+      ((setupDeps.elasticsearch.esNodesCompatibility$ as any) as BehaviorSubject<
+        NodesVersionCompatibility
+      >).next({
+        isCompatible: true,
+        incompatibleNodes: [],
+        warningNodes: [],
+        kibanaVersion: '8.0.0',
+      });
+      setImmediate(() => {
+        expect(mockKibanaMigratorInstance.runMigrations).toHaveBeenCalledTimes(1);
+        done();
+      });
     });
 
     it('resolves with KibanaMigrator after waiting for migrations to complete', async () => {
       const configService = configServiceMock.create({ atPath: { skip: false } });
       const coreContext = mockCoreContext.create({ configService });
       const soService = new SavedObjectsService(coreContext);
-      const coreSetup = ({
-        elasticsearch: { adminClient: { callAsInternalUser: jest.fn() } },
-        legacyPlugins: { uiExports: {}, pluginExtendedConfig: {} },
-      } as unknown) as SavedObjectsSetupDeps;
+      const coreSetup = createSetupDeps();
 
       await soService.setup(coreSetup);
       expect(mockKibanaMigratorInstance.runMigrations).toHaveBeenCalledTimes(0);
 
       const startContract = await soService.start({});
       expect(startContract.migrator).toBe(mockKibanaMigratorInstance);
-      expect(mockKibanaMigratorInstance.runMigrations).toHaveBeenCalledWith(false);
       expect(mockKibanaMigratorInstance.runMigrations).toHaveBeenCalledTimes(1);
     });
   });
diff --git a/src/core/server/saved_objects/saved_objects_service.ts b/src/core/server/saved_objects/saved_objects_service.ts
index 69ca8306ca4da..03b166c705126 100644
--- a/src/core/server/saved_objects/saved_objects_service.ts
+++ b/src/core/server/saved_objects/saved_objects_service.ts
@@ -18,7 +18,7 @@
  */
 
 import { CoreService } from 'src/core/types';
-import { first } from 'rxjs/operators';
+import { first, filter, take } from 'rxjs/operators';
 import {
   SavedObjectsClient,
   SavedObjectsSchema,
@@ -186,6 +186,7 @@ export class SavedObjectsService
   private migrator: KibanaMigrator | undefined;
   private logger: Logger;
   private clientProvider: ISavedObjectsClientProvider<KibanaRequest> | undefined;
+  private setupDeps: SavedObjectsSetupDeps | undefined;
 
   constructor(private readonly coreContext: CoreContext) {
     this.logger = coreContext.logger.get('savedobjects-service');
@@ -197,6 +198,8 @@ export class SavedObjectsService
   ): Promise<InternalSavedObjectsServiceSetup> {
     this.logger.debug('Setting up SavedObjects service');
 
+    this.setupDeps = setupDeps;
+
     const {
       savedObjectSchemas: savedObjectsSchemasDefinition,
       savedObjectMappings,
@@ -223,14 +226,14 @@ export class SavedObjectsService
       savedObjectMappings,
       savedObjectMigrations,
       savedObjectValidations,
-      logger: this.coreContext.logger.get('migrations'),
+      logger: this.logger,
       kibanaVersion: this.coreContext.env.packageInfo.version,
       config: setupDeps.legacyPlugins.pluginExtendedConfig,
       savedObjectsConfig,
       kibanaConfig,
       callCluster: migrationsRetryCallCluster(
         adminClient.callAsInternalUser,
-        this.coreContext.logger.get('migrations'),
+        this.logger,
         migrationsRetryDelay
       ),
     }));
@@ -287,7 +290,23 @@ export class SavedObjectsService
       .pipe(first())
       .toPromise();
     const skipMigrations = cliArgs.optimize || savedObjectsConfig.skip;
-    await this.migrator!.runMigrations(skipMigrations);
+
+    if (skipMigrations) {
+      this.logger.warn(
+        'Skipping Saved Object migrations on startup. Note: Individual documents will still be migrated when read or written.'
+      );
+    } else {
+      this.logger.info(
+        'Waiting until all Elasticsearch nodes are compatible with Kibana before starting saved objects migrations...'
+      );
+      await this.setupDeps!.elasticsearch.esNodesCompatibility$.pipe(
+        filter(nodes => nodes.isCompatible),
+        take(1)
+      ).toPromise();
+
+      this.logger.info('Starting saved objects migrations');
+      await this.migrator!.runMigrations();
+    }
 
     return {
       migrator: this.migrator!,
diff --git a/src/legacy/core_plugins/elasticsearch/index.d.ts b/src/legacy/core_plugins/elasticsearch/index.d.ts
index 4cbb1c82cc1e4..df713160137a6 100644
--- a/src/legacy/core_plugins/elasticsearch/index.d.ts
+++ b/src/legacy/core_plugins/elasticsearch/index.d.ts
@@ -523,6 +523,7 @@ export interface CallCluster {
 }
 
 export interface ElasticsearchPlugin {
+  status: { on: (status: string, cb: () => void) => void };
   getCluster(name: string): Cluster;
   createCluster(name: string, config: ClusterConfig): Cluster;
   waitUntilReady(): Promise<void>;
diff --git a/src/legacy/core_plugins/elasticsearch/index.js b/src/legacy/core_plugins/elasticsearch/index.js
index da7b557e7ea19..55e81e4144bb5 100644
--- a/src/legacy/core_plugins/elasticsearch/index.js
+++ b/src/legacy/core_plugins/elasticsearch/index.js
@@ -17,10 +17,10 @@
  * under the License.
  */
 import { first } from 'rxjs/operators';
-import healthCheck from './lib/health_check';
 import { Cluster } from './lib/cluster';
 import { createProxy } from './lib/create_proxy';
 import { handleESError } from './lib/handle_es_error';
+import { versionHealthCheck } from './lib/version_health_check';
 
 export default function(kibana) {
   let defaultVars;
@@ -92,15 +92,13 @@ export default function(kibana) {
 
       createProxy(server);
 
-      // Set up the health check service and start it.
-      const { start, waitUntilReady } = healthCheck(
+      const waitUntilHealthy = versionHealthCheck(
         this,
-        server,
-        esConfig.healthCheckDelay.asMilliseconds(),
-        esConfig.ignoreVersionMismatch
+        server.logWithMetadata,
+        server.newPlatform.__internals.elasticsearch.esNodesCompatibility$
       );
-      server.expose('waitUntilReady', waitUntilReady);
-      start();
+
+      server.expose('waitUntilReady', () => waitUntilHealthy);
     },
   });
 }
diff --git a/src/legacy/core_plugins/elasticsearch/integration_tests/elasticsearch.test.ts b/src/legacy/core_plugins/elasticsearch/integration_tests/elasticsearch.test.ts
new file mode 100644
index 0000000000000..5806c31b78414
--- /dev/null
+++ b/src/legacy/core_plugins/elasticsearch/integration_tests/elasticsearch.test.ts
@@ -0,0 +1,89 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import {
+  createTestServers,
+  TestElasticsearchUtils,
+  TestKibanaUtils,
+  TestUtils,
+  createRootWithCorePlugins,
+  getKbnServer,
+} from '../../../../test_utils/kbn_server';
+
+import { BehaviorSubject } from 'rxjs';
+// eslint-disable-next-line @kbn/eslint/no-restricted-paths
+import { NodesVersionCompatibility } from 'src/core/server/elasticsearch/version_check/ensure_es_version';
+
+describe('Elasticsearch plugin', () => {
+  let servers: TestUtils;
+  let esServer: TestElasticsearchUtils;
+  let root: TestKibanaUtils['root'];
+  let elasticsearch: TestKibanaUtils['kbnServer']['server']['plugins']['elasticsearch'];
+
+  const esNodesCompatibility$ = new BehaviorSubject<NodesVersionCompatibility>({
+    isCompatible: true,
+    incompatibleNodes: [],
+    warningNodes: [],
+    kibanaVersion: '8.0.0',
+  });
+
+  beforeAll(async function() {
+    const settings = {
+      elasticsearch: {},
+      adjustTimeout: (t: any) => {
+        jest.setTimeout(t);
+      },
+    };
+    servers = createTestServers(settings);
+    esServer = await servers.startES();
+
+    const elasticsearchSettings = {
+      hosts: esServer.hosts,
+      username: esServer.username,
+      password: esServer.password,
+    };
+    root = createRootWithCorePlugins({ elasticsearch: elasticsearchSettings });
+
+    const setup = await root.setup();
+    setup.elasticsearch.esNodesCompatibility$ = esNodesCompatibility$;
+    await root.start();
+
+    elasticsearch = getKbnServer(root).server.plugins.elasticsearch;
+  });
+
+  afterAll(async () => {
+    await esServer.stop();
+    await root.shutdown();
+  }, 30000);
+
+  it("should set it's status to green when all nodes are compatible", done => {
+    jest.setTimeout(30000);
+    elasticsearch.status.on('green', () => done());
+  });
+
+  it("should set it's status to red when some nodes aren't compatible", done => {
+    esNodesCompatibility$.next({
+      isCompatible: false,
+      incompatibleNodes: [],
+      warningNodes: [],
+      kibanaVersion: '8.0.0',
+    });
+    elasticsearch.status.on('red', () => done());
+  });
+});
diff --git a/src/legacy/core_plugins/elasticsearch/lib/__tests__/ensure_es_version.js b/src/legacy/core_plugins/elasticsearch/lib/__tests__/ensure_es_version.js
deleted file mode 100644
index 781d198c66236..0000000000000
--- a/src/legacy/core_plugins/elasticsearch/lib/__tests__/ensure_es_version.js
+++ /dev/null
@@ -1,223 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import _ from 'lodash';
-import Bluebird from 'bluebird';
-import sinon from 'sinon';
-import expect from '@kbn/expect';
-
-import { esTestConfig } from '@kbn/test';
-import { ensureEsVersion } from '../ensure_es_version';
-
-describe('plugins/elasticsearch', () => {
-  describe('lib/ensure_es_version', () => {
-    const KIBANA_VERSION = '5.1.0';
-
-    let server;
-
-    beforeEach(function() {
-      server = {
-        log: sinon.stub(),
-        logWithMetadata: sinon.stub(),
-        plugins: {
-          elasticsearch: {
-            getCluster: sinon
-              .stub()
-              .withArgs('admin')
-              .returns({ callWithInternalUser: sinon.stub() }),
-            status: {
-              red: sinon.stub(),
-            },
-            url: esTestConfig.getUrl(),
-          },
-        },
-        config() {
-          return {
-            get: sinon.stub(),
-          };
-        },
-      };
-    });
-
-    function setNodes(/* ...versions */) {
-      const versions = _.shuffle(arguments);
-      const nodes = {};
-      let i = 0;
-
-      while (versions.length) {
-        const name = 'node-' + ++i;
-        const version = versions.shift();
-
-        const node = {
-          version: version,
-          http: {
-            publish_address: 'http_address',
-          },
-          ip: 'ip',
-        };
-
-        if (!_.isString(version)) _.assign(node, version);
-        nodes[name] = node;
-      }
-
-      const cluster = server.plugins.elasticsearch.getCluster('admin');
-      cluster.callWithInternalUser
-        .withArgs('nodes.info', sinon.match.any)
-        .returns(Bluebird.resolve({ nodes: nodes }));
-    }
-
-    function setNodeWithoutHTTP(version) {
-      const nodes = { 'node-without-http': { version, ip: 'ip' } };
-      const cluster = server.plugins.elasticsearch.getCluster('admin');
-      cluster.callWithInternalUser
-        .withArgs('nodes.info', sinon.match.any)
-        .returns(Bluebird.resolve({ nodes: nodes }));
-    }
-
-    it('returns true with single a node that matches', async () => {
-      setNodes('5.1.0');
-      const result = await ensureEsVersion(server, KIBANA_VERSION);
-      expect(result).to.be(true);
-    });
-
-    it('returns true with multiple nodes that satisfy', async () => {
-      setNodes('5.1.0', '5.2.0', '5.1.1-Beta1');
-      const result = await ensureEsVersion(server, KIBANA_VERSION);
-      expect(result).to.be(true);
-    });
-
-    it('throws an error with a single node that is out of date', async () => {
-      // 5.0.0 ES is too old to work with a 5.1.0 version of Kibana.
-      setNodes('5.1.0', '5.2.0', '5.0.0');
-      try {
-        await ensureEsVersion(server, KIBANA_VERSION);
-      } catch (e) {
-        expect(e).to.be.a(Error);
-      }
-    });
-
-    it('does not throw on outdated nodes, if `ignoreVersionMismatch` is enabled in development mode', async () => {
-      // set config values
-      server.config = () => ({
-        get: name => {
-          switch (name) {
-            case 'env.dev':
-              return true;
-            default:
-              throw new Error(`Unknown option "${name}"`);
-          }
-        },
-      });
-
-      // 5.0.0 ES is too old to work with a 5.1.0 version of Kibana.
-      setNodes('5.1.0', '5.2.0', '5.0.0');
-
-      const ignoreVersionMismatch = true;
-      const result = await ensureEsVersion(server, KIBANA_VERSION, ignoreVersionMismatch);
-      expect(result).to.be(true);
-    });
-
-    it('throws an error if `ignoreVersionMismatch` is enabled in production mode', async () => {
-      // set config values
-      server.config = () => ({
-        get: name => {
-          switch (name) {
-            case 'env.dev':
-              return false;
-            default:
-              throw new Error(`Unknown option "${name}"`);
-          }
-        },
-      });
-
-      // 5.0.0 ES is too old to work with a 5.1.0 version of Kibana.
-      setNodes('5.1.0', '5.2.0', '5.0.0');
-
-      try {
-        const ignoreVersionMismatch = true;
-        await ensureEsVersion(server, KIBANA_VERSION, ignoreVersionMismatch);
-      } catch (e) {
-        expect(e).to.be.a(Error);
-      }
-    });
-
-    it('fails if that single node is a client node', async () => {
-      setNodes('5.1.0', '5.2.0', { version: '5.0.0', attributes: { client: 'true' } });
-      try {
-        await ensureEsVersion(server, KIBANA_VERSION);
-      } catch (e) {
-        expect(e).to.be.a(Error);
-      }
-    });
-
-    it('warns if a node is only off by a patch version', async () => {
-      setNodes('5.1.1');
-      await ensureEsVersion(server, KIBANA_VERSION);
-      sinon.assert.callCount(server.logWithMetadata, 2);
-      expect(server.logWithMetadata.getCall(0).args[0]).to.contain('debug');
-      expect(server.logWithMetadata.getCall(1).args[0]).to.contain('warning');
-    });
-
-    it('warns if a node is off by a patch version and without http publish address', async () => {
-      setNodeWithoutHTTP('5.1.1');
-      await ensureEsVersion(server, KIBANA_VERSION);
-      sinon.assert.callCount(server.logWithMetadata, 2);
-      expect(server.logWithMetadata.getCall(0).args[0]).to.contain('debug');
-      expect(server.logWithMetadata.getCall(1).args[0]).to.contain('warning');
-    });
-
-    it('errors if a node incompatible and without http publish address', async () => {
-      setNodeWithoutHTTP('6.1.1');
-      try {
-        await ensureEsVersion(server, KIBANA_VERSION);
-      } catch (e) {
-        expect(e.message).to.contain('incompatible nodes');
-        expect(e).to.be.a(Error);
-      }
-    });
-
-    it('only warns once per node list', async () => {
-      setNodes('5.1.1');
-
-      await ensureEsVersion(server, KIBANA_VERSION);
-      sinon.assert.callCount(server.logWithMetadata, 2);
-      expect(server.logWithMetadata.getCall(0).args[0]).to.contain('debug');
-      expect(server.logWithMetadata.getCall(1).args[0]).to.contain('warning');
-
-      await ensureEsVersion(server, KIBANA_VERSION);
-      sinon.assert.callCount(server.logWithMetadata, 3);
-      expect(server.logWithMetadata.getCall(2).args[0]).to.contain('debug');
-    });
-
-    it('warns again if the node list changes', async () => {
-      setNodes('5.1.1');
-
-      await ensureEsVersion(server, KIBANA_VERSION);
-      sinon.assert.callCount(server.logWithMetadata, 2);
-      expect(server.logWithMetadata.getCall(0).args[0]).to.contain('debug');
-      expect(server.logWithMetadata.getCall(1).args[0]).to.contain('warning');
-
-      setNodes('5.1.2');
-      await ensureEsVersion(server, KIBANA_VERSION);
-      sinon.assert.callCount(server.logWithMetadata, 4);
-      expect(server.logWithMetadata.getCall(2).args[0]).to.contain('debug');
-      expect(server.logWithMetadata.getCall(3).args[0]).to.contain('warning');
-    });
-  });
-});
diff --git a/src/legacy/core_plugins/elasticsearch/lib/__tests__/health_check.js b/src/legacy/core_plugins/elasticsearch/lib/__tests__/health_check.js
deleted file mode 100644
index 3b593c6352394..0000000000000
--- a/src/legacy/core_plugins/elasticsearch/lib/__tests__/health_check.js
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import Bluebird from 'bluebird';
-import sinon from 'sinon';
-import expect from '@kbn/expect';
-
-const NoConnections = require('elasticsearch').errors.NoConnections;
-
-import healthCheck from '../health_check';
-import kibanaVersion from '../kibana_version';
-
-const esPort = 9220;
-
-describe('plugins/elasticsearch', () => {
-  describe('lib/health_check', function() {
-    let health;
-    let plugin;
-    let cluster;
-    let server;
-    const sandbox = sinon.createSandbox();
-
-    function getTimerCount() {
-      return Object.keys(sandbox.clock.timers || {}).length;
-    }
-
-    beforeEach(() => {
-      sandbox.useFakeTimers();
-      const COMPATIBLE_VERSION_NUMBER = '5.0.0';
-
-      // Stub the Kibana version instead of drawing from package.json.
-      sandbox.stub(kibanaVersion, 'get').returns(COMPATIBLE_VERSION_NUMBER);
-
-      // setup the plugin stub
-      plugin = {
-        name: 'elasticsearch',
-        status: {
-          red: sinon.stub(),
-          green: sinon.stub(),
-          yellow: sinon.stub(),
-        },
-      };
-
-      cluster = { callWithInternalUser: sinon.stub(), errors: { NoConnections } };
-      cluster.callWithInternalUser.withArgs('index', sinon.match.any).returns(Bluebird.resolve());
-      cluster.callWithInternalUser
-        .withArgs('mget', sinon.match.any)
-        .returns(Bluebird.resolve({ ok: true }));
-      cluster.callWithInternalUser
-        .withArgs('get', sinon.match.any)
-        .returns(Bluebird.resolve({ found: false }));
-      cluster.callWithInternalUser
-        .withArgs('search', sinon.match.any)
-        .returns(Bluebird.resolve({ hits: { hits: [] } }));
-      cluster.callWithInternalUser.withArgs('nodes.info', sinon.match.any).returns(
-        Bluebird.resolve({
-          nodes: {
-            'node-01': {
-              version: COMPATIBLE_VERSION_NUMBER,
-              http_address: `inet[/127.0.0.1:${esPort}]`,
-              ip: '127.0.0.1',
-            },
-          },
-        })
-      );
-
-      // Setup the server mock
-      server = {
-        logWithMetadata: sinon.stub(),
-        info: { port: 5601 },
-        config: () => ({ get: sinon.stub() }),
-        plugins: {
-          elasticsearch: {
-            getCluster: sinon.stub().returns(cluster),
-          },
-        },
-        ext: sinon.stub(),
-      };
-
-      health = healthCheck(plugin, server, 0);
-    });
-
-    afterEach(() => sandbox.restore());
-
-    it('should stop when cluster is shutdown', () => {
-      // ensure that health.start() is responsible for the timer we are observing
-      expect(getTimerCount()).to.be(0);
-      health.start();
-      expect(getTimerCount()).to.be(1);
-
-      // ensure that a server extension was registered
-      sinon.assert.calledOnce(server.ext);
-      sinon.assert.calledWithExactly(server.ext, sinon.match.string, sinon.match.func);
-
-      const [, handler] = server.ext.firstCall.args;
-      handler(); // this should be health.stop
-
-      // ensure that the handler unregistered the timer
-      expect(getTimerCount()).to.be(0);
-    });
-
-    it('should set the cluster green if everything is ready', function() {
-      cluster.callWithInternalUser.withArgs('ping').returns(Bluebird.resolve());
-
-      return health.run().then(function() {
-        sinon.assert.calledOnce(plugin.status.yellow);
-        sinon.assert.calledWithExactly(plugin.status.yellow, 'Waiting for Elasticsearch');
-
-        sinon.assert.calledOnce(
-          cluster.callWithInternalUser.withArgs('nodes.info', sinon.match.any)
-        );
-        sinon.assert.notCalled(plugin.status.red);
-        sinon.assert.calledOnce(plugin.status.green);
-        sinon.assert.calledWithExactly(plugin.status.green, 'Ready');
-      });
-    });
-
-    describe('#waitUntilReady', function() {
-      it('waits for green status', function() {
-        plugin.status.once = sinon.spy(function(event, handler) {
-          expect(event).to.be('green');
-          setImmediate(handler);
-        });
-
-        const waitUntilReadyPromise = health.waitUntilReady();
-
-        sandbox.clock.runAll();
-
-        return waitUntilReadyPromise.then(function() {
-          sinon.assert.calledOnce(plugin.status.once);
-        });
-      });
-    });
-  });
-});
diff --git a/src/legacy/core_plugins/elasticsearch/lib/ensure_es_version.js b/src/legacy/core_plugins/elasticsearch/lib/ensure_es_version.js
deleted file mode 100644
index 8d304cd558418..0000000000000
--- a/src/legacy/core_plugins/elasticsearch/lib/ensure_es_version.js
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * ES and Kibana versions are locked, so Kibana should require that ES has the same version as
- * that defined in Kibana's package.json.
- */
-
-import { forEach, get } from 'lodash';
-import { coerce } from 'semver';
-import isEsCompatibleWithKibana from './is_es_compatible_with_kibana';
-
-/**
- * tracks the node descriptions that get logged in warnings so
- * that we don't spam the log with the same message over and over.
- *
- * There are situations, like in testing or multi-tenancy, where
- * the server argument changes, so we must track the previous
- * node warnings per server
- */
-const lastWarnedNodesForServer = new WeakMap();
-
-export function ensureEsVersion(server, kibanaVersion, ignoreVersionMismatch = false) {
-  const { callWithInternalUser } = server.plugins.elasticsearch.getCluster('admin');
-
-  server.logWithMetadata(['plugin', 'debug'], 'Checking Elasticsearch version');
-  return callWithInternalUser('nodes.info', {
-    filterPath: ['nodes.*.version', 'nodes.*.http.publish_address', 'nodes.*.ip'],
-  }).then(function(info) {
-    // Aggregate incompatible ES nodes.
-    const incompatibleNodes = [];
-
-    // Aggregate ES nodes which should prompt a Kibana upgrade.
-    const warningNodes = [];
-
-    forEach(info.nodes, esNode => {
-      if (!isEsCompatibleWithKibana(esNode.version, kibanaVersion)) {
-        // Exit early to avoid collecting ES nodes with newer major versions in the `warningNodes`.
-        return incompatibleNodes.push(esNode);
-      }
-
-      // It's acceptable if ES and Kibana versions are not the same so long as
-      // they are not incompatible, but we should warn about it
-
-      // Ignore version qualifiers
-      // https://github.com/elastic/elasticsearch/issues/36859
-      const looseMismatch = coerce(esNode.version).version !== coerce(kibanaVersion).version;
-      if (looseMismatch) {
-        warningNodes.push(esNode);
-      }
-    });
-
-    function getHumanizedNodeNames(nodes) {
-      return nodes.map(node => {
-        const publishAddress = get(node, 'http.publish_address')
-          ? get(node, 'http.publish_address') + ' '
-          : '';
-        return 'v' + node.version + ' @ ' + publishAddress + '(' + node.ip + ')';
-      });
-    }
-
-    if (warningNodes.length) {
-      const simplifiedNodes = warningNodes.map(node => ({
-        version: node.version,
-        http: {
-          publish_address: get(node, 'http.publish_address'),
-        },
-        ip: node.ip,
-      }));
-
-      // Don't show the same warning over and over again.
-      const warningNodeNames = getHumanizedNodeNames(simplifiedNodes).join(', ');
-      if (lastWarnedNodesForServer.get(server) !== warningNodeNames) {
-        lastWarnedNodesForServer.set(server, warningNodeNames);
-        server.logWithMetadata(
-          ['warning'],
-          `You're running Kibana ${kibanaVersion} with some different versions of ` +
-            'Elasticsearch. Update Kibana or Elasticsearch to the same ' +
-            `version to prevent compatibility issues: ${warningNodeNames}`,
-          {
-            kibanaVersion,
-            nodes: simplifiedNodes,
-          }
-        );
-      }
-    }
-
-    if (incompatibleNodes.length && !shouldIgnoreVersionMismatch(server, ignoreVersionMismatch)) {
-      const incompatibleNodeNames = getHumanizedNodeNames(incompatibleNodes);
-      throw new Error(
-        `This version of Kibana requires Elasticsearch v` +
-          `${kibanaVersion} on all nodes. I found ` +
-          `the following incompatible nodes in your cluster: ${incompatibleNodeNames.join(', ')}`
-      );
-    }
-
-    return true;
-  });
-}
-
-function shouldIgnoreVersionMismatch(server, ignoreVersionMismatch) {
-  const isDevMode = server.config().get('env.dev');
-  if (!isDevMode && ignoreVersionMismatch) {
-    throw new Error(
-      `Option "elasticsearch.ignoreVersionMismatch" can only be used in development mode`
-    );
-  }
-
-  return isDevMode && ignoreVersionMismatch;
-}
diff --git a/src/legacy/core_plugins/elasticsearch/lib/health_check.js b/src/legacy/core_plugins/elasticsearch/lib/health_check.js
deleted file mode 100644
index 40053ec774542..0000000000000
--- a/src/legacy/core_plugins/elasticsearch/lib/health_check.js
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import Bluebird from 'bluebird';
-import kibanaVersion from './kibana_version';
-import { ensureEsVersion } from './ensure_es_version';
-
-export default function(plugin, server, requestDelay, ignoreVersionMismatch) {
-  plugin.status.yellow('Waiting for Elasticsearch');
-
-  function waitUntilReady() {
-    return new Bluebird(resolve => {
-      plugin.status.once('green', resolve);
-    });
-  }
-
-  function check() {
-    return ensureEsVersion(server, kibanaVersion.get(), ignoreVersionMismatch)
-      .then(() => plugin.status.green('Ready'))
-      .catch(err => plugin.status.red(err));
-  }
-
-  let timeoutId = null;
-
-  function scheduleCheck(ms) {
-    if (timeoutId) return;
-
-    const myId = setTimeout(function() {
-      check().finally(function() {
-        if (timeoutId === myId) startorRestartChecking();
-      });
-    }, ms);
-
-    timeoutId = myId;
-  }
-
-  function startorRestartChecking() {
-    scheduleCheck(stopChecking() ? requestDelay : 1);
-  }
-
-  function stopChecking() {
-    if (!timeoutId) return false;
-    clearTimeout(timeoutId);
-    timeoutId = null;
-    return true;
-  }
-
-  server.ext('onPreStop', stopChecking);
-
-  return {
-    waitUntilReady: waitUntilReady,
-    run: check,
-    start: startorRestartChecking,
-    stop: stopChecking,
-    isRunning: function() {
-      return !!timeoutId;
-    },
-  };
-}
diff --git a/src/legacy/core_plugins/elasticsearch/lib/version_health_check.js b/src/legacy/core_plugins/elasticsearch/lib/version_health_check.js
new file mode 100644
index 0000000000000..4ee8307f490eb
--- /dev/null
+++ b/src/legacy/core_plugins/elasticsearch/lib/version_health_check.js
@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export const versionHealthCheck = (esPlugin, logWithMetadata, esNodesCompatibility$) => {
+  esPlugin.status.yellow('Waiting for Elasticsearch');
+
+  return new Promise(resolve => {
+    esNodesCompatibility$.subscribe(({ isCompatible, message, kibanaVersion, warningNodes }) => {
+      if (!isCompatible) {
+        esPlugin.status.red(message);
+      } else {
+        if (message) {
+          logWithMetadata(['warning'], message, {
+            kibanaVersion,
+            nodes: warningNodes,
+          });
+        }
+        esPlugin.status.green('Ready');
+        resolve();
+      }
+    });
+  });
+};
diff --git a/src/legacy/core_plugins/elasticsearch/lib/version_health_check.test.js b/src/legacy/core_plugins/elasticsearch/lib/version_health_check.test.js
new file mode 100644
index 0000000000000..ba7c95bcdfec5
--- /dev/null
+++ b/src/legacy/core_plugins/elasticsearch/lib/version_health_check.test.js
@@ -0,0 +1,71 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { versionHealthCheck } from './version_health_check';
+import { Subject } from 'rxjs';
+
+describe('plugins/elasticsearch', () => {
+  describe('lib/health_version_check', function() {
+    let plugin;
+    let logWithMetadata;
+
+    beforeEach(() => {
+      plugin = {
+        status: {
+          red: jest.fn(),
+          green: jest.fn(),
+          yellow: jest.fn(),
+        },
+      };
+
+      logWithMetadata = jest.fn();
+      jest.clearAllMocks();
+    });
+
+    it('returned promise resolves when all nodes are compatible ', function() {
+      const esNodesCompatibility$ = new Subject();
+      const versionHealthyPromise = versionHealthCheck(
+        plugin,
+        logWithMetadata,
+        esNodesCompatibility$
+      );
+      esNodesCompatibility$.next({ isCompatible: true, message: undefined });
+      return expect(versionHealthyPromise).resolves.toBe(undefined);
+    });
+
+    it('should set elasticsearch plugin status to green when all nodes are compatible', function() {
+      const esNodesCompatibility$ = new Subject();
+      versionHealthCheck(plugin, logWithMetadata, esNodesCompatibility$);
+      expect(plugin.status.yellow).toHaveBeenCalledWith('Waiting for Elasticsearch');
+      expect(plugin.status.green).not.toHaveBeenCalled();
+      esNodesCompatibility$.next({ isCompatible: true, message: undefined });
+      expect(plugin.status.green).toHaveBeenCalledWith('Ready');
+      expect(plugin.status.red).not.toHaveBeenCalled();
+    });
+
+    it('should set elasticsearch plugin status to red when some nodes are incompatible', function() {
+      const esNodesCompatibility$ = new Subject();
+      versionHealthCheck(plugin, logWithMetadata, esNodesCompatibility$);
+      expect(plugin.status.yellow).toHaveBeenCalledWith('Waiting for Elasticsearch');
+      expect(plugin.status.red).not.toHaveBeenCalled();
+      esNodesCompatibility$.next({ isCompatible: false, message: 'your nodes are incompatible' });
+      expect(plugin.status.red).toHaveBeenCalledWith('your nodes are incompatible');
+      expect(plugin.status.green).not.toHaveBeenCalled();
+    });
+  });
+});
diff --git a/src/legacy/server/http/integration_tests/default_route_provider.test.ts b/src/legacy/server/http/integration_tests/default_route_provider.test.ts
index 4898cb2b67852..d91438d904558 100644
--- a/src/legacy/server/http/integration_tests/default_route_provider.test.ts
+++ b/src/legacy/server/http/integration_tests/default_route_provider.test.ts
@@ -29,7 +29,7 @@ let mockDefaultRouteSetting: any = '';
 describe('default route provider', () => {
   let root: Root;
   beforeAll(async () => {
-    root = kbnTestServer.createRoot();
+    root = kbnTestServer.createRoot({ migrations: { skip: true } });
 
     await root.setup();
     await root.start();
diff --git a/src/legacy/server/http/integration_tests/default_route_provider_config.test.ts b/src/legacy/server/http/integration_tests/default_route_provider_config.test.ts
index da785a59893ab..8365941cbeb10 100644
--- a/src/legacy/server/http/integration_tests/default_route_provider_config.test.ts
+++ b/src/legacy/server/http/integration_tests/default_route_provider_config.test.ts
@@ -30,6 +30,7 @@ describe('default route provider', () => {
       server: {
         defaultRoute: '/app/some/default/route',
       },
+      migrations: { skip: true },
     });
 
     await root.setup();
diff --git a/src/legacy/server/http/integration_tests/max_payload_size.test.js b/src/legacy/server/http/integration_tests/max_payload_size.test.js
index 4408f0141bb21..7f22f83c78f0e 100644
--- a/src/legacy/server/http/integration_tests/max_payload_size.test.js
+++ b/src/legacy/server/http/integration_tests/max_payload_size.test.js
@@ -21,7 +21,7 @@ import * as kbnTestServer from '../../../../test_utils/kbn_server';
 
 let root;
 beforeAll(async () => {
-  root = kbnTestServer.createRoot({ server: { maxPayloadBytes: 100 } });
+  root = kbnTestServer.createRoot({ server: { maxPayloadBytes: 100 }, migrations: { skip: true } });
 
   await root.setup();
   await root.start();
diff --git a/x-pack/plugins/spaces/server/lib/request_interceptors/on_post_auth_interceptor.test.ts b/x-pack/plugins/spaces/server/lib/request_interceptors/on_post_auth_interceptor.test.ts
index c1f557f164ad6..b62211440bb7a 100644
--- a/x-pack/plugins/spaces/server/lib/request_interceptors/on_post_auth_interceptor.test.ts
+++ b/x-pack/plugins/spaces/server/lib/request_interceptors/on_post_auth_interceptor.test.ts
@@ -107,7 +107,10 @@ describe('onPostAuthInterceptor', () => {
     availableSpaces: any[],
     testOptions = { simulateGetSpacesFailure: false, simulateGetSingleSpaceFailure: false }
   ) {
-    const { http } = await root.setup();
+    const { http, elasticsearch } = await root.setup();
+
+    // Mock esNodesCompatibility$ to prevent `root.start()` from blocking on ES version check
+    elasticsearch.esNodesCompatibility$ = elasticsearchServiceMock.createInternalSetup().esNodesCompatibility$;
 
     const loggingMock = loggingServiceMock
       .create()
diff --git a/x-pack/plugins/spaces/server/lib/request_interceptors/on_request_interceptor.test.ts b/x-pack/plugins/spaces/server/lib/request_interceptors/on_request_interceptor.test.ts
index d6ff4a20052e4..5e6cf67ee8c90 100644
--- a/x-pack/plugins/spaces/server/lib/request_interceptors/on_request_interceptor.test.ts
+++ b/x-pack/plugins/spaces/server/lib/request_interceptors/on_request_interceptor.test.ts
@@ -17,6 +17,7 @@ import {
 
 import * as kbnTestServer from '../../../../../../src/test_utils/kbn_server';
 import { LegacyAPI } from '../../plugin';
+import { elasticsearchServiceMock } from 'src/core/server/mocks';
 
 describe('onRequestInterceptor', () => {
   let root: ReturnType<typeof kbnTestServer.createRoot>;
@@ -104,7 +105,9 @@ describe('onRequestInterceptor', () => {
     routes: 'legacy' | 'new-platform';
   }
   async function setup(opts: SetupOpts = { basePath: '/', routes: 'legacy' }) {
-    const { http } = await root.setup();
+    const { http, elasticsearch } = await root.setup();
+    // Mock esNodesCompatibility$ to prevent `root.start()` from blocking on ES version check
+    elasticsearch.esNodesCompatibility$ = elasticsearchServiceMock.createInternalSetup().esNodesCompatibility$;
 
     initSpacesOnRequestInterceptor({
       getLegacyAPI: () =>