feat(custom-setup): introduce custom npm registries (#542)

Co-authored-by: Roman Kuba <[email protected]>

.yarnrc

@@ -0,0 +1 @@
+--install.ignore-engines true

examples/cra/package.json

@@ -2,6 +2,8 @@
   "name": "cra",
   "version": "0.1.0",
   "private": true,
+  "license": "Apache-2.0",
+  "author": "CodeSandbox",
   "dependencies": {
     "@codesandbox/sandpack-react": "*",
     "@codesandbox/sandpack-themes": "*",

examples/custom-npm-registry/.gitignore

@@ -0,0 +1,2 @@
+node_modules
+storage

examples/custom-npm-registry/.yarnrc

@@ -0,0 +1 @@
+--install.ignore-engines true

examples/custom-npm-registry/README.md

@@ -0,0 +1,26 @@
+# Sandpack custom npm registry (proxy)
+
+This project works as a recipe to proxy NPM registries. The main goal is to enable the consumption of private packages and expose them through a private registry without any authentication token (GitHub or Npm) or requiring an authentication process. 
+
+This new private registry can be used on a Sandpack instance and run private packages public. 
+
+Disclaimer: it's essential to keep the information and tokens of the npm registry private! By using this method, it's best to keep in mind that it could expose all private packages in your account. Be careful where and how this proxy will be used. Make sure to use authentication tokens with **read-only access**.
+
+It's also possible to expose only specific packages. If the custom scopes are `@scope/package-name` instead of `@scope/*`, it will only expose that particular package. You can even do something like `@scope/design-system*` to expose all packages of the design system.
+
+## How it works
+This project relies on [Verdaccio](https://verdaccio.org/), an open-source project that creates a private registry and can proxy other registries, such as GitHub and Npm. 
+
+## How to use
+
+1. Host this project somewhere, and make sure it has permission to create new folders and files - Verdaccio needs to create temp storage to perform some optimizations;
+2. Configure your project correctly, for example, if you want to proxy NPM, GitHub, or both. You can find instructions in `/index.js`;
+3. Set the environments variables according to the type of registry you want to use;
+
+
+## Environment variables
+
+| Name | Description |
+| - | - |
+| `VERDACCIO_PUBLIC_URL` | is intended to be used behind proxies, and replace the final URL (optional) |
+| `GH_PKG_TOKEN` | GitHub personal token with `read:packages` permission |

examples/custom-npm-registry/htpasswd

@@ -0,0 +1 @@
+test:{SHA}qUqP5cyxm6YcTAhz05Hph5gvu9M=

examples/custom-npm-registry/index.js

@@ -0,0 +1,50 @@
+const { runServer } = require("@verdaccio/node-api");
+
+/**
+ * Custom configuration
+ */
+const customScopes = [`@codesandbox/*`];
+const registries = {
+  npmjs: {
+    url: "https://registry.npmjs.org/",
+    cache: false,
+  },
+  github: {
+    url: "https://npm.pkg.github.com/",
+    cache: false,
+    auth: {
+      type: "bearer",
+      token: process.env.GH_PKG_TOKEN,
+    },
+  },
+};
+
+/**
+ * Default configuration
+ */
+const defaultPermission = {
+  access: "$all",
+  publish: "$authenticated",
+  unpublish: "$authenticated",
+};
+const defaultConfiguration = {
+  config_path: "./config.yaml",
+  storage: "./storage",
+  uplinks: registries,
+  packages: {
+    ...customScopes.reduce((acc, scope) => {
+      acc[scope] = { ...defaultPermission, proxy: "github npmjs" };
+      return acc;
+    }, {}),
+    "@*/*": { ...defaultPermission, proxy: "npmjs" },
+    "**": { ...defaultPermission, proxy: "npmjs" },
+  },
+  server: { keepAliveTimeout: 60 },
+};
+
+(async () => {
+  const app = await runServer(defaultConfiguration);
+  app.listen(4000, () => {
+    console.log("server started");
+  });
+})();

examples/custom-npm-registry/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "custom-npm-registry",
+  "version": "1.0.0",
+  "description": "",
+  "dependencies": {
+    "@verdaccio/node-api": "^6.0.0-6-next.32",
+    "@verdaccio/ui-theme": "^3.4.1"
+  },
+  "license": "Apache-2.0",
+  "author": "CodeSandbox",
+  "devDependencies": {},
+  "scripts": {
+    "start": "node index.js"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/danilowoz/sandpack-proxy.git"
+  },
+  "bugs": {
+    "url": "https://github.com/danilowoz/sandpack-proxy/issues"
+  },
+  "homepage": "https://github.com/danilowoz/sandpack-proxy#readme"
+}

examples/gatsby/package.json

@@ -3,7 +3,8 @@
   "version": "1.0.0",
   "private": true,
   "description": "gatsby",
-  "author": "Danilo Woznica",
+  "license": "Apache-2.0",
+  "author": "CodeSandbox",
   "keywords": [
     "gatsby"
   ],

examples/nextjs/package.json

@@ -2,6 +2,8 @@
   "name": "nextjs",
   "private": true,
   "version": "0.0.1",
+  "license": "Apache-2.0",
+  "author": "CodeSandbox",
   "scripts": {
     "dev": "next dev",
     "build": "next build",

sandpack-client/src/client.ts

@@ -15,6 +15,7 @@ import type {
   SandpackError,
   ReactDevToolsMode,
   Template,
+  NpmRegistry,
 } from "./types";
 import {
   createPackageJSON,
@@ -76,6 +77,12 @@ export interface ClientOptions {
   };
 
   reactDevTools?: ReactDevToolsMode;
+
+  /**
+   * The custom private npm registry setting makes it possible
+   * to retrieve npm packages from your own npm registry.
+   */
+  customNpmRegistries?: NpmRegistry[];
 }
 
 export interface SandboxInfo {
@@ -306,6 +313,7 @@ export class SandpackClient {
       skipEval: this.options.skipEval || false,
       clearConsoleDisabled: !this.options.clearConsoleOnFirstCompile,
       logLevel: this.options.logLevel ?? SandpackLogLevel.Info,
+      customNpmRegistries: this.options.customNpmRegistries,
     });
   }
 

sandpack-client/src/types.ts

@@ -157,6 +157,18 @@ export interface ProtocolRequestMessage extends BaseProtocolMessage {
   params: any[];
 }
 
+export interface NpmRegistry {
+  enabledScopes: string[];
+  limitToScopes: boolean;
+  registryUrl: string;
+  /**
+   * It must be `false` if you're providing a sef-host solution,
+   * otherwise, it'll try to proxy from CodeSandbox Proxy
+   */
+  proxyEnabled: boolean;
+  registryAuthToken?: string;
+}
+
 export type SandpackMessage = BaseSandpackMessage &
   (
     | {
@@ -221,6 +233,7 @@ export type SandpackMessage = BaseSandpackMessage &
         clearConsoleDisabled?: boolean;
         reactDevTools?: ReactDevToolsMode;
         logLevel?: SandpackLogLevel;
+        customNpmRegistries?: NpmRegistry[];
       }
     | {
         type: "refresh";

sandpack-react/src/Playground.stories.tsx

@@ -31,7 +31,7 @@ export const Main = (): JSX.Element => {
       showNavigator: true,
       showRefreshButton: true,
     },
-    Template: "react",
+    Template: "react" as const,
     Theme: "auto",
   });
 

sandpack-react/src/contexts/sandpackContext.tsx

@@ -352,6 +352,13 @@ export class SandpackProviderClass extends React.PureComponent<
         showErrorScreen: !this.errorScreenRegistered.current,
         showLoadingScreen: !this.loadingScreenRegistered.current,
         reactDevTools: this.state.reactDevTools,
+        customNpmRegistries: this.props.customSetup?.npmRegistries?.map(
+          (config) =>
+            ({
+              ...config,
+              proxyEnabled: false, // force
+            } ?? [])
+        ),
       }
     );
 

sandpack-react/src/presets/CustomSandpack.stories.tsx

@@ -477,3 +477,31 @@ return <h1>{message}</h1>
     </SandpackLayout>
   </SandpackProvider>
 );
+
+export const CustomNpmRegistries: React.FC = () => (
+  <Sandpack
+    customSetup={{
+      dependencies: { "@codesandbox/test-package": "1.0.5" },
+      npmRegistries: [
+        {
+          enabledScopes: ["@codesandbox"],
+          limitToScopes: true,
+          registryUrl: "https://1gemwv-4000.preview.csb.app",
+        },
+      ],
+    }}
+    files={{
+      "/App.js": `import { Button } from "@codesandbox/test-package"
+
+export default function App() {
+  return (
+    <div>
+      <Button>I'm a private Package</Button>
+    </div>
+  )
+}
+`,
+    }}
+    template="react"
+  />
+);

sandpack-react/src/types.ts

@@ -10,6 +10,7 @@ import type {
   SandpackMessage,
   UnsubscribeFunction,
   SandpackLogLevel,
+  NpmRegistry,
 } from "@codesandbox/sandpack-client";
 import type React from "react";
 
@@ -194,6 +195,22 @@ export interface SandpackSetup {
    * they are designed to mirror the default behavior of the framework
    */
   environment?: SandboxEnvironment;
+
+  /**
+   * The custom private npm registry setting makes it possible
+   * to retrieve npm packages from your own npm registry.
+   *
+   * Examples:
+   * ```js
+   * {
+   *   enabledScopes: ["@codesandbox"],
+   *   registryUrl: "//my-registry.domain.com",
+   *   limitToScopes: true, // if false all packages will be fetched from custom registry
+   *   registryAuthToken: "SECRET" // optinal value, if public
+   * }
+   * ```
+   */
+  npmRegistries?: NpmRegistry[];
 }
 
 /**

website/docs/docs/faq.md

@@ -9,9 +9,13 @@ title: FAQ
 
 Sandpack is an open ecosystem of components and utilities that allow you to compile and run modern frameworks in the browser. You can either use one of our predefined `components` for embedding the *CodeSandbox* experience into your projects, or you can build your own version of `sandpack`, on top of our standard components and utilities.
 
-#### How to load local or private dependencies?
+#### How to load private dependencies?
 
-Currently, Sandpack doesn’t have a way to consume private dependencies from any kind of registry service, because the bundler host is shared with all Sandpack consumers apps. However, you can pass local dependencies just like a regular file or using the external resource API:
+Read the following [guide](/guides/private-packages).
+
+#### How to load local dependencies? 
+
+Currently, Sandpack doesn’t have a way to consume local dependencies, because the bundler host is shared with all Sandpack consumers apps. However, you can pass local dependencies just like a regular file or using the external resource API:
 
 ```jsx
 <Sandpack

website/docs/docs/guides/guides-overview.md

@@ -6,6 +6,7 @@ title: Guides Overview
 Here you can find some guides or recipes on using the Sandpack in different situations. These examples are basic implementations, and you need to consider improving for production, but you can find more detailed guides in the [advanced usage](/advanced-usage/provider) section.
 
 - Integrate [Monaco Editor](../guides/integrate-monaco-editor.md)
+- Integrate [private packages](../guides/private-packages.md)
 - Integrate Prettier for code formatting: [CodeSandbox example](https://codesandbox.io/s/sandpack-prettier-1po91?file=/src/App.js)
 - Integrate ESLint for static code analysis: [CodeSandbox example](https://codesandbox.io/s/sandpack-eslint-vztlt?file=/src/App.tsx)
-- Integrate Real Time collabration using firepad : [repo example](https://github.com/hussamkhatib/Real-time-collaborative-sandpack)
+- Integrate Real Time collabration using firepad: [repo example](https://github.com/hussamkhatib/Real-time-collaborative-sandpack)

website/docs/docs/guides/private-packages.md

@@ -0,0 +1,66 @@
+---
+sidebar_position: 2
+title: Private packages
+---
+
+<big>The custom private NPM registry allows Sandpack instances to retrieve private NPM packages from your own registry. This option requires running a third service (Node.js server) and configuring your Sandpack provider to consume these dependencies from another registry, not the public ones.</big>
+
+<br/>
+<br/>
+
+![Private packages](/img/private-package.png)
+
+<br/>
+
+**You'll need:**
+- Host a Node.js server, which will run registry proxy;
+- GitHub/NPM authentication token with read access;
+
+## Self-host the proxy
+
+We recommend hosting a service that allows you to proxy your private packages from a registry (GitHub/Npm/your own) to a new one, which would make the packages available through another URL. 
+As Sandpack bundles everything in-browser, it needs to find a way to connect to the registry which provides the project dependencies. 
+First, Sandpack will try to fetch all dependencies from public registries, for example, `react` or `redux`. Then you can let Sandpack know which dependencies (or scoped dependencies) should be fetched from a different registry. For example, your custom registry.
+
+### Our recommendation
+Suppose you don't already have a public registry, we recommend using [Verdaccio](https://verdaccio.org/). An open-source project that creates a private registry and can proxy other registries, such as GitHub and Npm. 
+You can find examples of how to use the [examples folder](https://github.com/codesandbox/sandpack/tree/main/examples) in the main repository.
+
+## Sandpack configuration
+
+Once the proxy is running and configured, you need to set some options in your Sandpack context:
+
+```jsx
+ <Sandpack
+    customSetup={{
+      dependencies: { "@codesandbox/test-package": "1.0.5" },
+      npmRegistries: [
+        {
+          enabledScopes: ["@codesandbox"],
+          limitToScopes: true,
+          registryUrl: "PROXY_URL", 
+        },
+      ],
+    }}
+    files={{
+      "/App.js": `import { Button } from "@codesandbox/test-package"
+
+export default function App() {
+  return (
+    <div>
+      <Button>I'm a private Package</Button>
+    </div>
+  )
+}
+`,
+    }}
+    template="react"
+  />
+```
+
+## Security
+It's essential to keep the information and tokens of the npm registry private! By using this method, it's best to keep in mind that it could expose all private packages in your account. Be careful where and how this proxy will be used. Make sure to use authentication tokens with **read-only access**.
+
+It's also possible to expose only specific packages. If the custom scopes are `@scope/package-name` instead of `@scope/*`, it will only expose that particular package. You can even do something like `@scope/design-system*` to expose all packages of the design system.
+
+