Add SSL support when starting Elasticsearch from yarn es (#42527)

* Initial work

* Add integration tests

* Use constants

* Fix broken code

* Handle scenario where esArgs is a string

* Remove || []

* Apply PR feedback

* Use const format

packages/kbn-dev-utils/package.json

@@ -6,7 +6,7 @@
   "license": "Apache-2.0",
   "private": true,
   "scripts": {
-    "build": "babel src --out-dir target",
+    "build": "babel src --out-dir target --copy-files",
     "kbn:bootstrap": "yarn build --quiet",
     "kbn:watch": "yarn build --watch"
   },

packages/kbn-dev-utils/src/certs/ca.crt

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDSjCCAjKgAwIBAgIVAOgxLlE1RMGl2fYgTKDznvDL2vboMA0GCSqGSIb3DQEB
+CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu
+ZXJhdGVkIENBMB4XDTE5MDcxMTE3MzQ0OFoXDTIyMDcxMDE3MzQ0OFowNDEyMDAG
+A1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5lcmF0ZWQgQ0Ew
+ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCNImKp/A9l++Ac7U5lvHOA
++fYRb8p7AgdfKBMB0v3bo+bpHjkbkf3vYHjo1xJSg5ls6EPK+Do4owkAgKJdrznI
+5/efJOjgA+ylH4rgAfrRIQmiFEWZnAv86vJ+Iq83mfkPELb4dvXCi7AFQkzoM/rY
+Lbi97xha5bA2SEmpYp7VhBTM9zWy+q9Tm5odPO8u2n75GpIM2RwipaXlL0ink+06
+/oweQJoivaDgpDOmUXCFPmpV3VCdhUGxDQPyG0upQkF+NbQoei4RmluPEmVz4S7I
+TFLWjX7LeZVP63bJkcCgiq6Hm97kDtr9EYlPKhHm7UMWzhNzHbfvySMDzqAJC0KX
+AgMBAAGjUzBRMB0GA1UdDgQWBBRKqaaQ/+jT+ipPLJe7qekp1N/zizAfBgNVHSME
+GDAWgBRKqaaQ/+jT+ipPLJe7qekp1N/zizAPBgNVHRMBAf8EBTADAQH/MA0GCSqG
+SIb3DQEBCwUAA4IBAQA7Gcq8h8yDXvepfKUAcTTMCBZkI+g3qE1gfRwjW7587CIj
+xnrzEqANU+Q1lv7IeQ158HiduDUMZfnvpuNwkf0HkqnRWb57RwfVdCAlAeZmzipq
+5ZJWlIW4dbmk57nGLg4fCszedi0uSGytZ2/BUdpWyC0fAM97h7Agtr4xGGKMEL67
+uB55ijt61V62HZ5wWXWNO9m+wfmdnt+YQViQJHtpYz1oOmWhY3dpitZLfWs1sLLD
+w3CZOhmWX7+P7+HlCkSBF4swzHOCI3THyX61NbLxju8VkTAjwbZPq4EOnVKnO6kr
+RdwQVnzKnqG5fxfSGknNahy0pOhJHZlGLwECRlgF
+-----END CERTIFICATE-----

packages/kbn-dev-utils/src/certs/elasticsearch.crt

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDRDCCAiygAwIBAgIVAI8V1fwvXKykKtp5k0cLpTOtY+DVMA0GCSqGSIb3DQEB
+CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu
+ZXJhdGVkIENBMB4XDTE5MDcxMTE3MzUxOFoXDTIyMDcxMDE3MzUxOFowGDEWMBQG
+A1UEAxMNZWxhc3RpY3NlYXJjaDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC
+ggEBALW+8gV6m6wYmTZmrXzNWKElE+ePkkikCviNfuWonWqxgAoWpAwAx2FvdhP3
+UDFbe38ydJX4oDgXeC25vdIR6z2uqzx+GXSNSybO7luuOUYQOP4Xf5Cj3zzXXMyu
+nY1nZTVsChI9jAMz4cZZdUd04f4r4TBNxrFCcVR0uec5RGRXuP8rSQd9AbYFUVYf
+jJeLb24asghb2Ku+c2JGvMqPEXFWFGOXFhUoIbRjCJNTDcr1ZXPof3+fO1l6HmhT
+QBSqC4IZL8XqANltDT4tCQDD8L9+ckWJD8MP3wPkPUGZId2gLu++hrb9YfiP2upq
+N/f3P7l5Fcisw1iwQC4+DGMTyfcCAwEAAaNpMGcwHQYDVR0OBBYEFGuiGk8HLpG2
+MyA24/J+GwxT32ikMB8GA1UdIwQYMBaAFEqpppD/6NP6Kk8sl7up6SnU3/OLMBoG
+A1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATAJBgNVHRMEAjAAMA0GCSqGSIb3DQEB
+CwUAA4IBAQB8yfY0edAgq2KnJNWyl8NpHNfqtM27+/LR2V8OxVwxV1hc4ZilczLu
+CXeqP9uqBVjcck6fvLrjy4LhSG0V05j51UMJ1FjFVTBuhlrDcd3j8848yWrmyz8z
+vPYYY2vIN9d1NsBgufULwliBT4UJchsYE8xT5ayAzGHKCTlzHGHMTPzYjwac8nbT
+nd2u+6h0OQOJn6K4v+RfXtN4EA8ZUrYxUkqHNS3cFB5sxH7JQGi25XJc5MfxyCwY
+YOukxbN85ew861N6oVd+W+nGJu8WOLU88/uvCv+dLhnAlnnIOLqvmrD5m7gFsFO9
+Z7Xz/U1SbNipWy9OLOhqq2Ja59j8p9e5
+-----END CERTIFICATE-----

packages/kbn-dev-utils/src/certs/elasticsearch.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAtb7yBXqbrBiZNmatfM1YoSUT54+SSKQK+I1+5aidarGAChak
+DADHYW92E/dQMVt7fzJ0lfigOBd4Lbm90hHrPa6rPH4ZdI1LJs7uW645RhA4/hd/
+kKPfPNdczK6djWdlNWwKEj2MAzPhxll1R3Th/ivhME3GsUJxVHS55zlEZFe4/ytJ
+B30BtgVRVh+Ml4tvbhqyCFvYq75zYka8yo8RcVYUY5cWFSghtGMIk1MNyvVlc+h/
+f587WXoeaFNAFKoLghkvxeoA2W0NPi0JAMPwv35yRYkPww/fA+Q9QZkh3aAu776G
+tv1h+I/a6mo39/c/uXkVyKzDWLBALj4MYxPJ9wIDAQABAoIBAQCb1ggrjn/gxo7I
+yK3FL0XplqNEkCR8SLxndtvyC+w+Schh3hv3dst+zlXOtOZ8C9cOr7KrzS2EKwuP
+GY6bi2XL0/NbwTwOZgCkXBahYfgWDV7w8DEfUoPd5UPa9XZ+gsOTVPolvcRKErhq
+nNYk2SHWEMXb5zSRVUlbg2LL0pzD88bIuKJX+FwPvWcQc2P4OdVTq77iedcl82zZ
+6PqTNqKMep7/odLQeBfX7OapOAviVnPYHe0TA114COOimR/pK8IA1OJymX5rgU7O
+Wh+uNBSxdHsTTYTkAvw8Bt5Q8n1WCpQwZoYU3xWuSlu7eJ7kcgdFOu9r9GjSXysT
+UYCd8s0BAoGBAPXPpCDRxjqF3/ToZ5x5dorKxxJyrmldzMJaUjqOv7y6kezbdBql
+n7p3AJ5UfYUW/N6pgQXaWF4MPSyj7ItHhwHjL+v0Manmi5gq8oA30fplhjUlPre7
+Lx4v7SEmH739EHrkZ2ClIQwY3wKuN8mZKgw6RseFgphczDmhHCqEbjW3AoGBAL1H
+fkl0RNdZ3nZg0u7MUVk8ytnqBsp7bNFhEs0zUl7ghu3NLaPt8qhirG638oMSCxqH
+FPeM3/DryokQAym+UHYNMwiBziEUB2CKMMj7S5YFFWIldCxFeImCO2EP+y3hmbTZ
+yjsznNrDzQtErZGP+JTRZcy9xF0oAfVt0G/O1Q3BAoGAa8bqINW5g6l1Q82uuEXt
+evdkB6uu21YcVE8D5Nb4LMjk+KRUKObbvQc2hzVmf7dPklVh0+4jdsEJBYyuR3dK
+M8KoHV3JdMQ4CrUx9JQFBjQDf0PgVvDEvQiogTNVEZlm42tIBHECp2o0RdmbblIw
+xIG8zPi2BRYTGWWRkvbT18sCgYA+c/B/XBW62LRGavwuPsw4nY5xCH7lIIRvMZB6
+lIyBMaRToneEt2ZxmN08SwWBqdpwDlIkvB7H54UUZGwmwdzaltBX5jyVPX6RpAck
+yYXPIi5EDAeg8+sptAbTp+pA4UdOHO5VSlpe9GwbY7XBabejotPsElFQS3sZ9/nm
+amByAQKBgQCJWghllys1qk76/6PmeVjwjaK9n8o+94LWhqODXlACmDRyse5dwpYb
+BIsMMZrNu1YsqDXlWpU7xNa6A8j4oa+EPnm/01PjdueAvMB/oE1woawM5tSsd8NQ
+zeQPDhxjDxzaO5l4oJLZg6FT7iQAprhYZjgb8m1vz0D2Xid0A3Kgpw==
+-----END RSA PRIVATE KEY-----

packages/kbn-dev-utils/src/certs/index.js

@@ -0,0 +1,24 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { resolve } from 'path';
+
+export const CA_CERT_PATH = resolve(__dirname, 'ca.crt');
+export const ES_KEY_PATH = resolve(__dirname, 'elasticsearch.key');
+export const ES_CERT_PATH = resolve(__dirname, 'elasticsearch.crt');

packages/kbn-dev-utils/src/index.js

@@ -20,3 +20,4 @@
 export { withProcRunner } from './proc_runner';
 export { ToolingLog, ToolingLogTextWriter, pickLevelFromFlags } from './tooling_log';
 export { createAbsolutePathSerializer } from './serializers';
+export { CA_CERT_PATH, ES_KEY_PATH, ES_CERT_PATH } from './certs';

packages/kbn-es/src/cli_commands/archive.js

@@ -36,6 +36,7 @@ exports.help = (defaults = {}) => {
       --install-path    Installation path, defaults to 'source' within base-path
       --password        Sets password for elastic user [default: ${password}]
       --password.[user] Sets password for native realm user [default: ${password}]
+      --ssl             Sets up SSL on Elasticsearch
       -E                Additional key=value settings to pass to Elasticsearch
 
     Example:
@@ -56,7 +57,7 @@ exports.run = async (defaults = {}) => {
     default: defaults,
   });
 
-  const cluster = new Cluster();
+  const cluster = new Cluster({ ssl: options.ssl });
   const [, path] = options._;
 
   if (!path || !path.endsWith('tar.gz')) {

packages/kbn-es/src/cli_commands/snapshot.js

@@ -38,6 +38,7 @@ exports.help = (defaults = {}) => {
       --password.[user] Sets password for native realm user [default: ${password}]
       -E                Additional key=value settings to pass to Elasticsearch
       --download-only   Download the snapshot but don't actually start it
+      --ssl             Sets up SSL on Elasticsearch
 
     Example:
 
@@ -62,7 +63,7 @@ exports.run = async (defaults = {}) => {
     default: defaults,
   });
 
-  const cluster = new Cluster();
+  const cluster = new Cluster({ ssl: options.ssl });
   if (options['download-only']) {
     await cluster.downloadSnapshot(options);
   } else {

packages/kbn-es/src/cli_commands/source.js

@@ -36,6 +36,7 @@ exports.help = (defaults = {}) => {
       --data-archive    Path to zip or tarball containing an ES data directory to seed the cluster with.
       --password        Sets password for elastic user [default: ${password}]
       --password.[user] Sets password for native realm user [default: ${password}]
+      --ssl             Sets up SSL on Elasticsearch
       -E                Additional key=value settings to pass to Elasticsearch
 
     Example:
@@ -58,7 +59,7 @@ exports.run = async (defaults = {}) => {
     default: defaults,
   });
 
-  const cluster = new Cluster();
+  const cluster = new Cluster({ ssl: options.ssl });
   const { installPath } = await cluster.installSource(options);
 
   if (options.dataArchive) {

packages/kbn-es/src/cluster.js

@@ -17,6 +17,8 @@
  * under the License.
  */
 
+const fs = require('fs');
+const util = require('util');
 const execa = require('execa');
 const chalk = require('chalk');
 const path = require('path');
@@ -33,6 +35,8 @@ const { createCliError } = require('./errors');
 const { promisify } = require('util');
 const treeKillAsync = promisify(require('tree-kill'));
 const { parseSettings, SettingsFilter } = require('./settings');
+const { CA_CERT_PATH, ES_KEY_PATH, ES_CERT_PATH } = require('@kbn/dev-utils');
+const readFile = util.promisify(fs.readFile);
 
 // listen to data on stream until map returns anything but undefined
 const first = (stream, map) =>
@@ -48,8 +52,10 @@ const first = (stream, map) =>
   });
 
 exports.Cluster = class Cluster {
-  constructor(log = defaultLog) {
+  constructor({ log = defaultLog, ssl = false } = {}) {
     this._log = log;
+    this._ssl = ssl;
+    this._caCertPromise = ssl ? readFile(CA_CERT_PATH) : undefined;
   }
 
   /**
@@ -251,10 +257,18 @@ exports.Cluster = class Cluster {
     this._log.info(chalk.bold('Starting'));
     this._log.indent(4);
 
-    const args = parseSettings(
-      extractConfigFiles(options.esArgs || [], installPath, { log: this._log }),
-      { filter: SettingsFilter.NonSecureOnly }
-    ).reduce(
+    // Add to esArgs if ssl is enabled
+    const esArgs = [].concat(options.esArgs || []);
+    if (this._ssl) {
+      esArgs.push('xpack.security.http.ssl.enabled=true');
+      esArgs.push(`xpack.security.http.ssl.key=${ES_KEY_PATH}`);
+      esArgs.push(`xpack.security.http.ssl.certificate=${ES_CERT_PATH}`);
+      esArgs.push(`xpack.security.http.ssl.certificate_authorities=${CA_CERT_PATH}`);
+    }
+
+    const args = parseSettings(extractConfigFiles(esArgs, installPath, { log: this._log }), {
+      filter: SettingsFilter.NonSecureOnly,
+    }).reduce(
       (acc, [settingName, settingValue]) => acc.concat(['-E', `${settingName}=${settingValue}`]),
       []
     );
@@ -282,7 +296,14 @@ exports.Cluster = class Cluster {
 
     // once the http port is available setup the native realm
     this._nativeRealmSetup = httpPort.then(async port => {
-      const nativeRealm = new NativeRealm(options.password, port, this._log);
+      const caCert = await this._caCertPromise;
+      const nativeRealm = new NativeRealm({
+        port,
+        caCert,
+        log: this._log,
+        elasticPassword: options.password,
+        ssl: this._ssl,
+      });
       await nativeRealm.setPasswords(options);
     });
 

packages/kbn-es/src/integration_tests/__fixtures__/es_bin.js

@@ -19,9 +19,11 @@
  * under the License.
  */
 
-const { createServer } = require('http');
+const fs = require('fs');
 const { format: formatUrl } = require('url');
-const { exitCode, start } = JSON.parse(process.argv[2]);
+const { exitCode, start, ssl } = JSON.parse(process.argv[2]);
+const { createServer } = ssl ? require('https') : require('http');
+const { ES_KEY_PATH, ES_CERT_PATH } = require('@kbn/dev-utils');
 
 process.exitCode = exitCode;
 
@@ -30,27 +32,33 @@ if (!start) {
 }
 
 let serverUrl;
-const server = createServer((req, res) => {
-  const url = new URL(req.url, serverUrl);
-  const send = (code, body) => {
-    res.writeHead(code, { 'content-type': 'application/json' });
-    res.end(JSON.stringify(body));
-  };
+const server = createServer(
+  {
+    key: ssl ? fs.readFileSync(ES_KEY_PATH) : undefined,
+    cert: ssl ? fs.readFileSync(ES_CERT_PATH) : undefined,
+  },
+  (req, res) => {
+    const url = new URL(req.url, serverUrl);
+    const send = (code, body) => {
+      res.writeHead(code, { 'content-type': 'application/json' });
+      res.end(JSON.stringify(body));
+    };
 
-  if (url.pathname === '/_xpack') {
-    return send(400, {
+    if (url.pathname === '/_xpack') {
+      return send(400, {
+        error: {
+          reason: 'foo bar',
+        },
+      });
+    }
+
+    return send(404, {
       error: {
-        reason: 'foo bar',
+        reason: 'not found',
       },
     });
   }
-
-  return send(404, {
-    error: {
-      reason: 'not found',
-    },
-  });
-});
+);
 
 // setup server auto close after 1 second of silence
 let serverCloseTimer;