Pyodide Conversion: fix requirements handling

doc/how_to/wasm/convert.md

@@ -9,19 +9,21 @@ The ``panel convert`` command has the following options:
 
     options:
     -h, --help            show this help message and exit
-    --to TO               The format to convert to, one of 'pyodide' (default), 'pyodide-worker' or 'pyscript'
+    --to TO               The format to convert to, one of 'pyodide' (default), 'pyodide-worker', 'pyscript' or 'pyscript-worker'
     --compiled            Whether to use the compiled and faster version of Pyodide.
     --out OUT             The directory to write the file to.
     --title TITLE         A custom title for the application(s).
     --skip-embed          Whether to skip embedding pre-rendered content in the converted file to display content while app is loading.
     --index               Whether to create an index if multiple files are served.
     --pwa                 Whether to add files to serve applications as a Progressive Web App.
     --requirements REQUIREMENTS [REQUIREMENTS ...]
-                            Explicit requirements to add to the converted file, a single requirements.txt file or a JSON file containing requirements per app. By default requirements are inferred from the code.
+                          Explicit requirements to add to the converted file, a single requirements.txt file or a JSON file containing requirements per app. By default requirements are inferred from the code.
+    --resources RESOURCES [RESOURCES ...]
+                          Files to pack for distribution with the app. Does only support files located in the directory of the main panel app (or in subdirectories below).
     --disable-http-patch  Whether to disable patching http requests using the pyodide-http library.
     --watch               Watch the files
     --num-procs NUM_PROCS
-                            The number of processes to start in parallel to convert the apps.
+                          The number of processes to start in parallel to convert the apps.
 
 ## Example
 
@@ -89,6 +91,7 @@ Using the `--to` argument on the CLI you can control the format of the file that
 - **`pyodide`** (default): Run application using Pyodide running in the main thread. This option is less performant than pyodide-worker but produces completely standalone HTML files that do not have to be hosted on a static file server (e.g. Github Pages).
 - **`pyodide-worker`**: Generates an HTML file and a JS file containing a Web Worker that runs in a separate thread. This is the most performant option, but files have to be hosted on a static file server.
 - **`pyscript`**: Generates an HTML leveraging PyScript. This produces standalone HTML files containing `<py-env>` and `<py-script>` tags containing the dependencies and the application code. This output is the most readable, and should have equivalent performance to the `pyodide` option.
+- **`pyscript-worker`**: Generates an HTML file and a separate PY file containing the panel app leveraging PyScript. This needs proper setup of Content-Security-Policies on the webserver.
 
 ## Requirements
 
@@ -104,6 +107,8 @@ Alternatively you may also provide a `requirements.txt` file:
 panel convert script.py --to pyodide-worker --out pyodide --requirements requirements.txt
 ```
 
+One also can provide URLs to Python wheels provided at filesystem or online locations. Wheels available in the local filesystem will be packed into a zip-file (which needs to be hosted with the app) which will be unpacked to emscriptens MEMFS for installation.
+
 ## Index
 
 If you convert multiple applications at once you may want to add an index to be able to navigate between the applications easily. To enable the index simply pass `--index` to the convert command.

doc/how_to/wasm/index.md

@@ -53,8 +53,8 @@ Note that since Panel is built on Bokeh server and Tornado it is also possible t
 :hidden:
 :maxdepth: 2
 
-convert
 standalone
+convert
 sphinx
 jupyterlite
 ```

panel/_templates/pyodide_worker.js

@@ -13,29 +13,29 @@ async function startApplication() {
   self.postMessage({type: 'status', msg: 'Loading pyodide'})
   self.pyodide = await loadPyodide();
   self.pyodide.globals.set("sendPatch", sendPatch);
-  console.log("Loaded!");
+  console.log("Loaded pyodide!");
+  const data_archives = [{{ data_archives }}];
+  for (const archive of data_archives) {
+    let zipResponse = await fetch(archive);
+    let zipBinary = await zipResponse.arrayBuffer();
+    self.postMessage({type: 'status', msg: `Unpacking ${archive}`})
+    self.pyodide.unpackArchive(zipBinary, "zip");
+  }
   await self.pyodide.loadPackage("micropip");
-  const env_spec = [{{ env_spec }}]
-  for (const pkg of env_spec) {
-    let pkg_name;
-    if (pkg.endsWith('.whl')) {
-      pkg_name = pkg.split('/').slice(-1)[0].split('-')[0]
-    } else {
-      pkg_name = pkg
-    }
-    self.postMessage({type: 'status', msg: `Installing ${pkg_name}`})
-    try {
-      await self.pyodide.runPythonAsync(`
-        import micropip
-        await micropip.install('${pkg}');
-      `);
-    } catch(e) {
-      console.log(e)
-      self.postMessage({
-	type: 'status',
-	msg: `Error while installing ${pkg_name}`
-      });
-    }
+  self.postMessage({type: 'status', msg: `Installing packages`});
+  // a finegrained approach installing dependencies one after another with status updates was implemented previously
+  // it somehow did not resolve previously installed dependencies correctly
+  try {
+    await pyodide.runPythonAsync(`
+    import micropip
+    await micropip.install([{{ env_spec }}]);
+    `);
+  } catch(e) {
+    console.log(e)
+    self.postMessage({
+      type: 'status',
+      msg: `Error while installing packages`
+  });
   }
   console.log("Packages loaded!");
   self.postMessage({type: 'status', msg: 'Executing code'})

panel/command/convert.py

@@ -28,7 +28,7 @@ class Convert(Subcommand):
         ('--to', dict(
             action  = 'store',
             type    = str,
-            help    = "The format to convert to, one of 'pyodide' (default), 'pyodide-worker' or 'pyscript'",
+            help    = "The format to convert to, one of 'pyodide' (default), 'pyodide-worker', 'pyscript' or 'pyscript-worker'",
             default = 'pyodide'
         )),
         ('--compiled', dict(
@@ -70,6 +70,12 @@ class Convert(Subcommand):
             action  = 'store_false',
             help    = "Whether to disable patching http requests using the pyodide-http library."
         )),
+        ('--resources', dict(
+            nargs   = '+',
+            help    = (
+                "Files to pack for distribution with the app. Does only support files located in the directory of the main panel app (or in subdirectories below)."
+            )
+        )),
         ('--watch', dict(
             action  = 'store_true',
             help    = "Watch the files"
@@ -112,10 +118,10 @@ def invoke(self, args: argparse.Namespace) -> None:
             try:
                 convert_apps(
                     files, dest_path=args.out, runtime=runtime, requirements=requirements,
-                    prerender=not args.skip_embed, build_index=index, build_pwa=args.pwa,
-                    title=args.title, max_workers=args.num_procs,
-                    http_patch=not args.disable_http_patch, compiled=args.compiled,
-                    verbose=True
+                    resources=args.resources, prerender=not args.skip_embed,
+                    build_index=index, build_pwa=args.pwa, title=args.title,
+                    max_workers=args.num_procs, http_patch=not args.disable_http_patch,
+                    compiled=args.compiled, verbose=True
                 )
             except KeyboardInterrupt:
                 print("Aborted while building docs.")  # noqa: T201