From 6aaed710ceef2c38adb5a7b26128c35672bb6c62 Mon Sep 17 00:00:00 2001 From: Joel Chen Date: Mon, 23 Mar 2020 12:53:33 -0700 Subject: [PATCH] feat: https support for dev reverse proxy (#1572) --- docs/guides/local-https-setup.md | 142 +++++++ .../subapp-server/lib/setup-hapi-routes.js | 19 +- packages/subapp-server/lib/utils.js | 30 +- packages/subapp-util/lib/index.js | 22 +- packages/subapp-web/lib/load.js | 55 ++- packages/xarc-app-dev/.eslintrc | 30 -- packages/xarc-app-dev/config/archetype.js | 124 +----- packages/xarc-app-dev/config/dev-proxy.js | 90 +++++ packages/xarc-app-dev/config/env-app.js | 17 + packages/xarc-app-dev/config/env-babel.js | 42 ++ packages/xarc-app-dev/config/env-karma.js | 11 + packages/xarc-app-dev/config/env-proxy.js | 16 + packages/xarc-app-dev/config/env-webpack.js | 50 +++ packages/xarc-app-dev/config/user-config.js | 7 + .../xarc-app-dev/lib/app-dev-middleware.js | 2 +- .../lib/dev-admin/admin-server.js | 5 +- .../xarc-app-dev/lib/dev-admin/log-parser.js | 8 +- .../xarc-app-dev/lib/dev-admin/log-reader.js | 12 +- .../xarc-app-dev/lib/dev-admin/middleware.js | 15 +- .../lib/dev-admin/redbird-proxy.js | 382 ++++++++++-------- .../lib/dev-admin/redbird-spawn.js | 125 ++++-- packages/xarc-app-dev/lib/utils.js | 15 +- packages/xarc-app-dev/package.json | 40 +- packages/xarc-app-dev/test/.eslintrc | 3 - packages/xarc-app-dev/test/mocha.opts | 2 - .../test/spec/dev-admin/log-parser.spec.js | 55 ++- packages/xarc-app-dev/xclap.js | 2 +- packages/xarc-app/arch-clap.js | 10 + 28 files changed, 888 insertions(+), 443 deletions(-) create mode 100644 docs/guides/local-https-setup.md delete mode 100644 packages/xarc-app-dev/.eslintrc create mode 100644 packages/xarc-app-dev/config/dev-proxy.js create mode 100644 packages/xarc-app-dev/config/env-app.js create mode 100644 packages/xarc-app-dev/config/env-babel.js create mode 100644 packages/xarc-app-dev/config/env-karma.js create mode 100644 packages/xarc-app-dev/config/env-proxy.js create mode 100644 packages/xarc-app-dev/config/env-webpack.js create mode 100644 packages/xarc-app-dev/config/user-config.js delete mode 100644 packages/xarc-app-dev/test/.eslintrc delete mode 100644 packages/xarc-app-dev/test/mocha.opts diff --git a/docs/guides/local-https-setup.md b/docs/guides/local-https-setup.md new file mode 100644 index 000000000..378b4c15f --- /dev/null +++ b/docs/guides/local-https-setup.md @@ -0,0 +1,142 @@ +# Localhost HTTPS Setup + +These instructions are for MacOS, verified on macOS Mojave version 10.14.6 + +## SSL Key and Certificate + +1. Generate SSL key and cert + +Copy and paste these commands in the terminal to run them. + +> Make sure to change the hostname `dev.mydomain.com` in both places to your desired value. + +```bash +openssl req -new -x509 -nodes -sha256 -days 3650 \ + -newkey rsa:2048 -out dev-proxy.crt -keyout dev-proxy.key \ + -extensions SAN -reqexts SAN -subj /CN=dev.mydomain.com \ + -config <(cat /etc/ssl/openssl.cnf \ + <(printf '[ext]\nbasicConstraints=critical,CA:TRUE,pathlen:0\n') \ + <(printf '[SAN]\nsubjectAltName=DNS:dev.mydomain.com,IP:127.0.0.1\n')) +``` + +2. Add cert to your system keychain + +```bash +sudo security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain dev-proxy.crt +``` + +3. Put the files `dev-proxy.key` and `dev-proxy.crt` in your app's dir. + + > Alternatively, you can put them in one of the directories listed below: + + - `src` + - `test` + - `config` + +## Development in HTTPS + +After everything's setup, you can start development in HTTPS with the following steps: + +1. Using your favorite editor, add this line to your `/etc/hosts` file. + + > Change the hostname `dev.mydomain.com` accordingly if you used a different one. + +``` +127.0.0.1 dev.mydomain.com +``` + +2. Now to run app dev in HTTPS, set the env `ELECTRODE_DEV_HTTPS` to `8443` and `HOST` to the domain name you created your cert for. + + - Example: `HOST=dev.mydomain.com ELECTRODE_DEV_HTTPS=8443 npm run dev` + + - And point your browser to `https://dev.mydomain.com:8443` + + - If you have access to listen on the standard HTTPS port `443`, then you can set it to `443` or `true`, and use the URL `https://dev.mydomain.com` directly. + + - Another way to trigger HTTPS is with the env `PORT`. If that is set to `443` exactly, then the dev proxy will enter HTTPS mode even if env `ELECTRODE_DEV_HTTPS` is not set. + +## Elevated Privilege + +Generally, normal users can't run program to listen on network port below 1024. + +> but that seems to have changed for MacOS Mojave https://news.ycombinator.com/item?id=18302380 + +So if you want to set the dev proxy to listen on the standard HTTP port 80 or HTTPS port 443, you might need to give it elevated access. + +The recommended approach to achieve this is to run the dev proxy in a separate terminal with elevated access: + +```bash +sudo HOST=dev.mydomain.com PORT=443 npx clap dev-proxy +``` + +And then start normal development in another terminal: + +```bash +HOST=dev.mydomain.com npm run dev +``` + +### Automatic Elevating (optional) + +Optional: for best result, please use the manual approach recommended above. + +If your machine requires elevated access for the proxy to listen at a port, then a dialog will pop up to ask you for your password. This is achieved with the module https://www.npmjs.com/package/sudo-prompt + +This requirement is automatically detected, but if you want to explicitly trigger the elevated access, you can set the env `ELECTRODE_DEV_ELEVATED` to `true`. + +> However, due to restrictions with acquiring elevated access, this automatic acquisition has quirks. For example, the logs from the dev proxy can't be shown in your console. + +## Custom Proxy Rules + +The dev proxy is using a slightly modified version of [redbird] with some fixes and enhancements that are pending PR merge. + +You can provide your own proxy rules with a file `dev-proxy-rules.js` in one of these directories: + +- `src` +- `test` +- `config` + +The file should export a function `setupRules`, like the example below: + +```js +export function setupRules(proxy, options) { + const { host, port, protocol } = options; + proxy.register( + `${protocol}://${host}:${port}/myapi/foo-api`, + `https://api.myserver.com/myapi/foo-api` + ); +} +``` + +Where: + +- `proxy` - the redbird proxy instance. +- `options` - all configuration for the proxy: + + - `host` - hostname proxy is using. + - `port` - primary port proxy is listening on. + - `appPort` - app server's port the proxy forward to. + - `httpPort` - HTTP port proxy is listening on. + - `httpsPort` - Port for HTTPS if it is enabled. + - `https` - `true`/`false` to indicate if proxy is running in HTTPS mode. + - `webpackDev` - `true`/`false` to indicate if running with webpack dev. + - `webpackDevPort` - webpack dev server's port the proxy is forwarding to. + - `webpackDevPort` - webpack dev server's host the proxy is forwarding to. + - `protocol` - primary protocol: `"http"` or `"https"`. + - `elevated` - `true`/`false` to indicate if proxy should acquire elevate access. + +- The primary protocol is `https` if HTTPS is enabled, else it's `http`. + +- `appPort` is the port your app server is expected to listen on. It's determined as follows: + +1. env `APP_SERVER_PORT` or `APP_PORT_FOR_PROXY` if it's a valid number. +2. fallback to `3100`. + +- Even if HTTPS is enabled, the proxy always listens on HTTP also. In that case, `httpPort` is determined as follows: + +1. env `PORT` if it's defined +2. if `appPort` is not `3000`, then fallback to `3000`. +3. finally fallback to `3300`. + +The primary API to register your proxy rule is [`proxy.register`](https://www.npmjs.com/package/redbird#redbirdregistersrc-target-opts). + +[redbird]: https://www.npmjs.com/package/redbird diff --git a/packages/subapp-server/lib/setup-hapi-routes.js b/packages/subapp-server/lib/setup-hapi-routes.js index bc3636332..f6f8440ff 100644 --- a/packages/subapp-server/lib/setup-hapi-routes.js +++ b/packages/subapp-server/lib/setup-hapi-routes.js @@ -8,7 +8,6 @@ const _ = require("lodash"); const Fs = require("fs"); const Path = require("path"); const assert = require("assert"); -const Url = require("url"); const util = require("util"); const optionalRequire = require("optional-require")(require); const scanDir = require("filter-scan-dir"); @@ -139,11 +138,9 @@ async function setupRoutesFromFile(srcDir, server, pluginOpts) { return h.continue; }); - topOpts.devBundleBase = Url.format({ - protocol: topOpts.devServer.https ? "https" : "http", - hostname: topOpts.devServer.host, - port: topOpts.devServer.port, - pathname: "/js/" + topOpts.devBundleBase = subAppUtil.formUrl({ + ..._.pick(topOpts.devServer, ["protocol", "host", "port"]), + path: "/js/" }); // register routes @@ -192,7 +189,7 @@ async function setupRoutesFromFile(srcDir, server, pluginOpts) { .type("text/html; charset=UTF-8") .code(200); } else if (HttpStatus.redirect[status]) { - return h.redirect(data.path); + return h.redirect(data.path).code(status); } else if (HttpStatus.displayHtml[status]) { return h.response(data.html !== undefined ? data.html : data).code(status); } else if (status >= 200 && status < 300) { @@ -256,11 +253,9 @@ async function setupRoutesFromDir(server, pluginOpts, fromDir) { }); } - topOpts.devBundleBase = Url.format({ - protocol: topOpts.devServer.https ? "https" : "http", - hostname: topOpts.devServer.host, - port: topOpts.devServer.port, - pathname: "/js/" + topOpts.devBundleBase = subAppUtil.formUrl({ + ..._.pick(topOpts.devServer, ["protocol", "host", "port"]), + path: "/js/" }); registerRoutes({ routes, topOpts, server }); diff --git a/packages/subapp-server/lib/utils.js b/packages/subapp-server/lib/utils.js index 6349be871..3b0ab1b9e 100644 --- a/packages/subapp-server/lib/utils.js +++ b/packages/subapp-server/lib/utils.js @@ -4,6 +4,13 @@ const Fs = require("fs"); const Path = require("path"); +const optionalRequire = require("optional-require")(require); +const { + settings = {}, + devServer = {}, + fullDevServer = {}, + httpDevServer = {} +} = optionalRequire("@xarc/app-dev/config/dev-proxy", { default: {} }); /** * Tries to import bundle chunk selector function if the corresponding option is set in the @@ -55,33 +62,22 @@ const updateFullTemplate = (baseDir, options) => { } }; -function findEnv(keys, defVal) { - const k = [].concat(keys).find(x => x && process.env.hasOwnProperty(x)); - return k ? process.env[k] : defVal; -} - function getDefaultRouteOptions() { - const isDevProxy = process.env.hasOwnProperty("APP_SERVER_PORT"); - const webpackDev = process.env.WEBPACK_DEV === "true"; + const { webpackDev, useDevProxy } = settings; // temporary location to write build artifacts in dev mode const buildArtifacts = ".etmp"; return { pageTitle: "Untitled Electrode Web Application", - // webpackDev, - isDevProxy, - // - devServer: { - host: findEnv([isDevProxy && "HOST", "WEBPACK_DEV_HOST", "WEBPACK_HOST"], "127.0.0.1"), - port: findEnv([isDevProxy && "PORT", "WEBPACK_DEV_PORT"], isDevProxy ? "3000" : "2992"), - https: Boolean(process.env.WEBPACK_DEV_HTTPS) - }, - // + useDevProxy, + devServer, + fullDevServer, + httpDevServer, stats: webpackDev ? `${buildArtifacts}/stats.json` : "dist/server/stats.json", iconStats: "dist/server/iconstats.json", criticalCSS: "dist/js/critical.css", buildArtifacts, - prodBundleBase: "/js/", + prodBundleBase: "/js", devBundleBase: "/js", cspNonceValue: undefined, templateFile: Path.join(__dirname, "..", "resources", "index-page") diff --git a/packages/subapp-util/lib/index.js b/packages/subapp-util/lib/index.js index 64b48a966..1e8156200 100644 --- a/packages/subapp-util/lib/index.js +++ b/packages/subapp-util/lib/index.js @@ -2,6 +2,7 @@ /* eslint-disable no-console, no-process-exit, max-params */ +const Url = require("url"); const Path = require("path"); const assert = require("assert"); const optionalRequire = require("optional-require")(require); @@ -289,6 +290,24 @@ function refreshAllSubApps() { } } +const formUrl = ({ protocol = "http", host = "", port = "", path = "" }) => { + let proto = protocol.toString().toLowerCase(); + let host2 = host; + + if (port) { + const sp = port.toString(); + if (sp === "80") { + proto = "http"; + } else if (sp === "443") { + proto = "https"; + } else if (host) { + host2 = `${host}:${port}`; + } + } + + return Url.format({ protocol: proto, host: host2, pathname: path }); +}; + module.exports = { es6Require, scanSubAppsFromDir, @@ -300,5 +319,6 @@ module.exports = { loadSubAppByName, loadSubAppServerByName, refreshSubAppByName, - refreshAllSubApps + refreshAllSubApps, + formUrl }; diff --git a/packages/subapp-web/lib/load.js b/packages/subapp-web/lib/load.js index 50bf5d16f..71493a6ac 100644 --- a/packages/subapp-web/lib/load.js +++ b/packages/subapp-web/lib/load.js @@ -1,6 +1,6 @@ "use strict"; -/* eslint-disable max-statements, no-console, complexity */ +/* eslint-disable max-statements, no-console, complexity, no-magic-numbers */ /* * - Figure out all the dependencies and bundles a subapp needs and make sure @@ -19,20 +19,34 @@ const retrieveUrl = require("request"); const util = require("./util"); const xaa = require("xaa"); const jsesc = require("jsesc"); -const { loadSubAppByName, loadSubAppServerByName } = require("subapp-util"); +const { loadSubAppByName, loadSubAppServerByName, formUrl } = require("subapp-util"); // global name to store client subapp runtime, ie: window.xarcV1 // V1: version 1. const xarc = "window.xarcV1"; // Size threshold of initial state string to embed it as a application/json script tag -// It's more efficent to JSON.parse large JSON data instead of embedding them as JS. +// It's more efficient to JSON.parse large JSON data instead of embedding them as JS. // https://quipblog.com/efficiently-loading-inlined-json-data-911960b0ac0a // > The data sizes are as follows: large is 1.7MB of JSON, medium is 130K, // > small is 10K and tiny is 781 bytes. const INITIAL_STATE_SIZE_FOR_JSON = 1024; let INITIAL_STATE_TAG_ID = 0; +const makeDevDebugMessage = (msg, reportLink = true) => { + const reportMsg = reportLink + ? `\nError: Please capture this info and submit a bug report at https://github.com/electrode-io/electrode` + : ""; + return `Error: at ${util.removeCwd(__filename)} +${msg}${reportMsg}`; +}; + +const makeDevDebugHtml = msg => { + return `

DEV ERROR

+

${msg}

+`; +}; + module.exports = function setup(setupContext, { props: setupProps }) { // TODO: create JSON schema to validate props @@ -61,10 +75,18 @@ module.exports = function setup(setupContext, { props: setupProps }) { // to inline in the index page. // const retrieveDevServerBundle = async () => { - return new Promise((resolve, reject) => { - retrieveUrl(`${bundleBase}${bundleAsset.name}`, (err, resp, body) => { - if (err) { - reject(err); + return new Promise(resolve => { + const routeOptions = setupContext.routeOptions; + const path = `${bundleBase}${bundleAsset.name}`; + const bundleUrl = formUrl({ ...routeOptions.httpDevServer, path }); + retrieveUrl(bundleUrl, (err, resp, body) => { + if (err || resp.statusCode !== 200) { + const msg = makeDevDebugMessage( + `Error: fail to retrieve subapp bundle from '${bundleUrl}' for inlining in index HTML +${err || body}` + ); + console.error(msg); // eslint-disable-line + resolve(makeDevDebugHtml(msg)); } else { resolve(``); } @@ -81,7 +103,7 @@ module.exports = function setup(setupContext, { props: setupProps }) { let inlineSubAppJs; const prepareSubAppJsBundle = () => { - const webpackDev = process.env.WEBPACK_DEV === "true"; + const { webpackDev } = setupContext.routeOptions; if (setupProps.inlineScript === "always" || (setupProps.inlineScript === true && !webpackDev)) { if (!webpackDev) { @@ -93,7 +115,9 @@ module.exports = function setup(setupContext, { props: setupProps }) { } else if (ext === ".css") { inlineSubAppJs = ``; } else { - inlineSubAppJs = ``; + const msg = makeDevDebugMessage(`Error: UNKNOWN bundle extension ${name}`); + console.error(msg); // eslint-disable-line + inlineSubAppJs = makeDevDebugHtml(msg); } } else { inlineSubAppJs = true; @@ -251,12 +275,13 @@ ${dynInitialState}