feat: https support for dev reverse proxy (#1572)

docs/guides/local-https-setup.md

@@ -0,0 +1,142 @@
+# Localhost HTTPS Setup
+
+These instructions are for MacOS, verified on macOS Mojave version 10.14.6
+
+## SSL Key and Certificate
+
+1. Generate SSL key and cert
+
+Copy and paste these commands in the terminal to run them.
+
+> Make sure to change the hostname `dev.mydomain.com` in both places to your desired value.
+
+```bash
+openssl req -new -x509 -nodes -sha256 -days 3650 \
+ -newkey rsa:2048 -out dev-proxy.crt -keyout dev-proxy.key \
+ -extensions SAN -reqexts SAN -subj /CN=dev.mydomain.com \
+ -config <(cat /etc/ssl/openssl.cnf \
+    <(printf '[ext]\nbasicConstraints=critical,CA:TRUE,pathlen:0\n') \
+    <(printf '[SAN]\nsubjectAltName=DNS:dev.mydomain.com,IP:127.0.0.1\n'))
+```
+
+2. Add cert to your system keychain
+
+```bash
+sudo security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain dev-proxy.crt
+```
+
+3.  Put the files `dev-proxy.key` and `dev-proxy.crt` in your app's dir.
+
+    > Alternatively, you can put them in one of the directories listed below:
+
+    - `src`
+    - `test`
+    - `config`
+
+## Development in HTTPS
+
+After everything's setup, you can start development in HTTPS with the following steps:
+
+1.  Using your favorite editor, add this line to your `/etc/hosts` file.
+
+    > Change the hostname `dev.mydomain.com` accordingly if you used a different one.
+
+```
+127.0.0.1       dev.mydomain.com
+```
+
+2.  Now to run app dev in HTTPS, set the env `ELECTRODE_DEV_HTTPS` to `8443` and `HOST` to the domain name you created your cert for.
+
+    - Example: `HOST=dev.mydomain.com ELECTRODE_DEV_HTTPS=8443 npm run dev`
+
+    - And point your browser to `https://dev.mydomain.com:8443`
+
+    - If you have access to listen on the standard HTTPS port `443`, then you can set it to `443` or `true`, and use the URL `https://dev.mydomain.com` directly.
+
+    - Another way to trigger HTTPS is with the env `PORT`. If that is set to `443` exactly, then the dev proxy will enter HTTPS mode even if env `ELECTRODE_DEV_HTTPS` is not set.
+
+## Elevated Privilege
+
+Generally, normal users can't run program to listen on network port below 1024.
+
+> but that seems to have changed for MacOS Mojave https://news.ycombinator.com/item?id=18302380
+
+So if you want to set the dev proxy to listen on the standard HTTP port 80 or HTTPS port 443, you might need to give it elevated access.
+
+The recommended approach to achieve this is to run the dev proxy in a separate terminal with elevated access:
+
+```bash
+sudo HOST=dev.mydomain.com PORT=443 npx clap dev-proxy
+```
+
+And then start normal development in another terminal:
+
+```bash
+HOST=dev.mydomain.com npm run dev
+```
+
+### Automatic Elevating (optional)
+
+Optional: for best result, please use the manual approach recommended above.
+
+If your machine requires elevated access for the proxy to listen at a port, then a dialog will pop up to ask you for your password. This is achieved with the module https://www.npmjs.com/package/sudo-prompt
+
+This requirement is automatically detected, but if you want to explicitly trigger the elevated access, you can set the env `ELECTRODE_DEV_ELEVATED` to `true`.
+
+> However, due to restrictions with acquiring elevated access, this automatic acquisition has quirks. For example, the logs from the dev proxy can't be shown in your console.
+
+## Custom Proxy Rules
+
+The dev proxy is using a slightly modified version of [redbird] with some fixes and enhancements that are pending PR merge.
+
+You can provide your own proxy rules with a file `dev-proxy-rules.js` in one of these directories:
+
+- `src`
+- `test`
+- `config`
+
+The file should export a function `setupRules`, like the example below:
+
+```js
+export function setupRules(proxy, options) {
+  const { host, port, protocol } = options;
+  proxy.register(
+    `${protocol}://${host}:${port}/myapi/foo-api`,
+    `https://api.myserver.com/myapi/foo-api`
+  );
+}
+```
+
+Where:
+
+- `proxy` - the redbird proxy instance.
+- `options` - all configuration for the proxy:
+
+  - `host` - hostname proxy is using.
+  - `port` - primary port proxy is listening on.
+  - `appPort` - app server's port the proxy forward to.
+  - `httpPort` - HTTP port proxy is listening on.
+  - `httpsPort` - Port for HTTPS if it is enabled.
+  - `https` - `true`/`false` to indicate if proxy is running in HTTPS mode.
+  - `webpackDev` - `true`/`false` to indicate if running with webpack dev.
+  - `webpackDevPort` - webpack dev server's port the proxy is forwarding to.
+  - `webpackDevPort` - webpack dev server's host the proxy is forwarding to.
+  - `protocol` - primary protocol: `"http"` or `"https"`.
+  - `elevated` - `true`/`false` to indicate if proxy should acquire elevate access.
+
+- The primary protocol is `https` if HTTPS is enabled, else it's `http`.
+
+- `appPort` is the port your app server is expected to listen on. It's determined as follows:
+
+1. env `APP_SERVER_PORT` or `APP_PORT_FOR_PROXY` if it's a valid number.
+2. fallback to `3100`.
+
+- Even if HTTPS is enabled, the proxy always listens on HTTP also. In that case, `httpPort` is determined as follows:
+
+1. env `PORT` if it's defined
+2. if `appPort` is not `3000`, then fallback to `3000`.
+3. finally fallback to `3300`.
+
+The primary API to register your proxy rule is [`proxy.register`](https://www.npmjs.com/package/redbird#redbirdregistersrc-target-opts).
+
+[redbird]: https://www.npmjs.com/package/redbird

packages/subapp-server/lib/setup-hapi-routes.js

@@ -8,7 +8,6 @@ const _ = require("lodash");
 const Fs = require("fs");
 const Path = require("path");
 const assert = require("assert");
-const Url = require("url");
 const util = require("util");
 const optionalRequire = require("optional-require")(require);
 const scanDir = require("filter-scan-dir");
@@ -139,11 +138,9 @@ async function setupRoutesFromFile(srcDir, server, pluginOpts) {
     return h.continue;
   });
 
-  topOpts.devBundleBase = Url.format({
-    protocol: topOpts.devServer.https ? "https" : "http",
-    hostname: topOpts.devServer.host,
-    port: topOpts.devServer.port,
-    pathname: "/js/"
+  topOpts.devBundleBase = subAppUtil.formUrl({
+    ..._.pick(topOpts.devServer, ["protocol", "host", "port"]),
+    path: "/js/"
   });
 
   // register routes
@@ -192,7 +189,7 @@ async function setupRoutesFromFile(srcDir, server, pluginOpts) {
             .type("text/html; charset=UTF-8")
             .code(200);
         } else if (HttpStatus.redirect[status]) {
-          return h.redirect(data.path);
+          return h.redirect(data.path).code(status);
         } else if (HttpStatus.displayHtml[status]) {
           return h.response(data.html !== undefined ? data.html : data).code(status);
         } else if (status >= 200 && status < 300) {
@@ -256,11 +253,9 @@ async function setupRoutesFromDir(server, pluginOpts, fromDir) {
     });
   }
 
-  topOpts.devBundleBase = Url.format({
-    protocol: topOpts.devServer.https ? "https" : "http",
-    hostname: topOpts.devServer.host,
-    port: topOpts.devServer.port,
-    pathname: "/js/"
+  topOpts.devBundleBase = subAppUtil.formUrl({
+    ..._.pick(topOpts.devServer, ["protocol", "host", "port"]),
+    path: "/js/"
   });
 
   registerRoutes({ routes, topOpts, server });

packages/subapp-server/lib/utils.js

@@ -4,6 +4,13 @@
 
 const Fs = require("fs");
 const Path = require("path");
+const optionalRequire = require("optional-require")(require);
+const {
+  settings = {},
+  devServer = {},
+  fullDevServer = {},
+  httpDevServer = {}
+} = optionalRequire("@xarc/app-dev/config/dev-proxy", { default: {} });
 
 /**
  * Tries to import bundle chunk selector function if the corresponding option is set in the
@@ -55,33 +62,22 @@ const updateFullTemplate = (baseDir, options) => {
   }
 };
 
-function findEnv(keys, defVal) {
-  const k = [].concat(keys).find(x => x && process.env.hasOwnProperty(x));
-  return k ? process.env[k] : defVal;
-}
-
 function getDefaultRouteOptions() {
-  const isDevProxy = process.env.hasOwnProperty("APP_SERVER_PORT");
-  const webpackDev = process.env.WEBPACK_DEV === "true";
+  const { webpackDev, useDevProxy } = settings;
   // temporary location to write build artifacts in dev mode
   const buildArtifacts = ".etmp";
   return {
     pageTitle: "Untitled Electrode Web Application",
-    //
     webpackDev,
-    isDevProxy,
-    //
-    devServer: {
-      host: findEnv([isDevProxy && "HOST", "WEBPACK_DEV_HOST", "WEBPACK_HOST"], "127.0.0.1"),
-      port: findEnv([isDevProxy && "PORT", "WEBPACK_DEV_PORT"], isDevProxy ? "3000" : "2992"),
-      https: Boolean(process.env.WEBPACK_DEV_HTTPS)
-    },
-    //
+    useDevProxy,
+    devServer,
+    fullDevServer,
+    httpDevServer,
     stats: webpackDev ? `${buildArtifacts}/stats.json` : "dist/server/stats.json",
     iconStats: "dist/server/iconstats.json",
     criticalCSS: "dist/js/critical.css",
     buildArtifacts,
-    prodBundleBase: "/js/",
+    prodBundleBase: "/js",
     devBundleBase: "/js",
     cspNonceValue: undefined,
     templateFile: Path.join(__dirname, "..", "resources", "index-page")

packages/subapp-util/lib/index.js

@@ -2,6 +2,7 @@
 
 /* eslint-disable no-console, no-process-exit, max-params */
 
+const Url = require("url");
 const Path = require("path");
 const assert = require("assert");
 const optionalRequire = require("optional-require")(require);
@@ -289,6 +290,24 @@ function refreshAllSubApps() {
   }
 }
 
+const formUrl = ({ protocol = "http", host = "", port = "", path = "" }) => {
+  let proto = protocol.toString().toLowerCase();
+  let host2 = host;
+
+  if (port) {
+    const sp = port.toString();
+    if (sp === "80") {
+      proto = "http";
+    } else if (sp === "443") {
+      proto = "https";
+    } else if (host) {
+      host2 = `${host}:${port}`;
+    }
+  }
+
+  return Url.format({ protocol: proto, host: host2, pathname: path });
+};
+
 module.exports = {
   es6Require,
   scanSubAppsFromDir,
@@ -300,5 +319,6 @@ module.exports = {
   loadSubAppByName,
   loadSubAppServerByName,
   refreshSubAppByName,
-  refreshAllSubApps
+  refreshAllSubApps,
+  formUrl
 };

packages/subapp-web/lib/load.js

@@ -1,6 +1,6 @@
 "use strict";
 
-/* eslint-disable max-statements, no-console, complexity */
+/* eslint-disable max-statements, no-console, complexity, no-magic-numbers */
 
 /*
  * - Figure out all the dependencies and bundles a subapp needs and make sure
@@ -19,20 +19,34 @@ const retrieveUrl = require("request");
 const util = require("./util");
 const xaa = require("xaa");
 const jsesc = require("jsesc");
-const { loadSubAppByName, loadSubAppServerByName } = require("subapp-util");
+const { loadSubAppByName, loadSubAppServerByName, formUrl } = require("subapp-util");
 
 // global name to store client subapp runtime, ie: window.xarcV1
 // V1: version 1.
 const xarc = "window.xarcV1";
 
 // Size threshold of initial state string to embed it as a application/json script tag
-// It's more efficent to JSON.parse large JSON data instead of embedding them as JS.
+// It's more efficient to JSON.parse large JSON data instead of embedding them as JS.
 // https://quipblog.com/efficiently-loading-inlined-json-data-911960b0ac0a
 // > The data sizes are as follows: large is 1.7MB of JSON, medium is 130K,
 // > small is 10K and tiny is 781 bytes.
 const INITIAL_STATE_SIZE_FOR_JSON = 1024;
 let INITIAL_STATE_TAG_ID = 0;
 
+const makeDevDebugMessage = (msg, reportLink = true) => {
+  const reportMsg = reportLink
+    ? `\nError: Please capture this info and submit a bug report at https://github.com/electrode-io/electrode`
+    : "";
+  return `Error: at ${util.removeCwd(__filename)}
+${msg}${reportMsg}`;
+};
+
+const makeDevDebugHtml = msg => {
+  return `<h1 style="background-color: red">DEV ERROR</h1>
+<p><pre style="color: red">${msg}</pre></p>
+<!-- ${msg} -->`;
+};
+
 module.exports = function setup(setupContext, { props: setupProps }) {
   // TODO: create JSON schema to validate props
 
@@ -61,10 +75,18 @@ module.exports = function setup(setupContext, { props: setupProps }) {
   // to inline in the index page.
   //
   const retrieveDevServerBundle = async () => {
-    return new Promise((resolve, reject) => {
-      retrieveUrl(`${bundleBase}${bundleAsset.name}`, (err, resp, body) => {
-        if (err) {
-          reject(err);
+    return new Promise(resolve => {
+      const routeOptions = setupContext.routeOptions;
+      const path = `${bundleBase}${bundleAsset.name}`;
+      const bundleUrl = formUrl({ ...routeOptions.httpDevServer, path });
+      retrieveUrl(bundleUrl, (err, resp, body) => {
+        if (err || resp.statusCode !== 200) {
+          const msg = makeDevDebugMessage(
+            `Error: fail to retrieve subapp bundle from '${bundleUrl}' for inlining in index HTML
+${err || body}`
+          );
+          console.error(msg); // eslint-disable-line
+          resolve(makeDevDebugHtml(msg));
         } else {
           resolve(`<script>/*${name}*/${body}</script>`);
         }
@@ -81,7 +103,7 @@ module.exports = function setup(setupContext, { props: setupProps }) {
   let inlineSubAppJs;
 
   const prepareSubAppJsBundle = () => {
-    const webpackDev = process.env.WEBPACK_DEV === "true";
+    const { webpackDev } = setupContext.routeOptions;
 
     if (setupProps.inlineScript === "always" || (setupProps.inlineScript === true && !webpackDev)) {
       if (!webpackDev) {
@@ -93,7 +115,9 @@ module.exports = function setup(setupContext, { props: setupProps }) {
         } else if (ext === ".css") {
           inlineSubAppJs = `<style id="${name}">${src}</style>`;
         } else {
-          inlineSubAppJs = `<!-- UNKNOWN bundle extension ${name} -->`;
+          const msg = makeDevDebugMessage(`Error: UNKNOWN bundle extension ${name}`);
+          console.error(msg); // eslint-disable-line
+          inlineSubAppJs = makeDevDebugHtml(msg);
         }
       } else {
         inlineSubAppJs = true;
@@ -251,12 +275,13 @@ ${dynInitialState}<script>${xarc}.startSubAppOnLoad({
       const handleError = err => {
         if (process.env.NODE_ENV !== "production") {
           const stack = util.removeCwd(err.stack);
-          console.error(`SSR subapp ${name} failed <error>${stack}</error>`); // eslint-disable-line
-          outputSpot.add(`<!-- SSR subapp ${name} failed
-
-${stack}
-
--->`);
+          const msg = makeDevDebugMessage(
+            `Error: SSR subapp ${name} failed
+${stack}`,
+            false // SSR failure is likely an issue in user code, don't show link to report bug
+          );
+          console.error(msg); // eslint-disable-line
+          outputSpot.add(makeDevDebugHtml(msg));
         } else if (request && request.log) {
           request.log(["error"], { msg: `SSR subapp ${name} failed`, err });
         }