Skip to content

Commit

Permalink
Improve piplite.install time for Panelite notebooks (#4388)
Browse files Browse the repository at this point in the history
  • Loading branch information
MarcSkovMadsen authored Mar 7, 2023
1 parent c9702ee commit f0c64ae
Show file tree
Hide file tree
Showing 16 changed files with 236 additions and 52 deletions.
2 changes: 1 addition & 1 deletion examples/gallery/apis/stocks_altair.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"\n",
"from bokeh.sampledata import stocks\n",
"\n",
"pn.extension('vega', template='fast-list')"
"pn.extension('vega', template='fast')"
]
},
{
Expand Down
2 changes: 1 addition & 1 deletion examples/gallery/apis/stocks_hvplot.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"\n",
"from bokeh.sampledata import stocks\n",
"\n",
"pn.extension(template='fast-list')"
"pn.extension(template='fast')"
]
},
{
Expand Down
2 changes: 1 addition & 1 deletion examples/gallery/apis/stocks_matplotlib.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"\n",
"from bokeh.sampledata import stocks\n",
"\n",
"pn.extension(template='fast-list')"
"pn.extension(template='fast')"
]
},
{
Expand Down
2 changes: 1 addition & 1 deletion examples/gallery/apis/stocks_plotly.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"\n",
"from bokeh.sampledata import stocks\n",
"\n",
"pn.extension('plotly', template='fast-list')"
"pn.extension('plotly', template='fast')"
]
},
{
Expand Down
2 changes: 1 addition & 1 deletion examples/gallery/components/LeafletHeatMap.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@
"metadata": {},
"outputs": [],
"source": [
"url=\"http://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_month.csv\"\n",
"url=\"https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_month.csv\"\n",
"earthquakes = pd.read_csv(url)\n",
"\n",
"heatmap = LeafletHeatMap(\n",
Expand Down
14 changes: 10 additions & 4 deletions examples/reference/indicators/Tqdm.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,12 @@
"\n",
"def run_loop(*events, timeout=0.2):\n",
" for i in tqdm(range(0,10), desc=\"My loop bar\", leave=True, colour='#666666'):\n",
" time.sleep(timeout)\n",
" if pn.state._is_pyodide:\n",
" # time.sleep does not work in pyodide\n",
" np.random.random((10**6, 30)) \n",
" else:\n",
" time.sleep(timeout)\n",
" \n",
"run_loop(timeout=0.01)\n",
"\n",
"tqdm"
]
},
Expand Down Expand Up @@ -105,7 +107,11 @@
"def run_nested_loop(*events, timeout=0.05):\n",
" for i in tqdm_outer(range(10)):\n",
" for j in tqdm_inner(range(10)):\n",
" time.sleep(timeout)\n",
" if pn.state._is_pyodide:\n",
" # time.sleep does not work in pyodide\n",
" np.random.random((10**6, 30)) \n",
" else:\n",
" time.sleep(timeout)\n",
" \n",
"run_nested_loop(timeout=0.01)\n",
"\n",
Expand Down
7 changes: 6 additions & 1 deletion examples/reference/panes/DataFrame.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,12 @@
"source": [
"## Streamz DataFrames\n",
"\n",
"In addition to rendering standard pandas `DataFrame` and `Series` types the `DataFrame` pane will also render updating `streamz` types (Note: in a live kernel you should see the dataframe update every 0.5 seconds):"
"In addition to rendering standard pandas `DataFrame` and `Series` types the `DataFrame` pane will also render updating `streamz` types.",
"\n",
"Note:\n",
"\n",
"- In a live kernel you should see the dataframe update every 0.5 seconds.\n",
"- `streamz` does currently not work in Pyodide/PyScript."
]
},
{
Expand Down
2 changes: 1 addition & 1 deletion examples/reference/panes/ECharts.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"The ECharts pane also hass support for pyecharts. For example, we can pass a `pyecharts.charts.Bar` chart directly the `ECharts` pane. "
"The ECharts pane also hass support for pyecharts. For example, we can pass a `pyecharts.charts.Bar` chart directly the `ECharts` pane."
]
},
{
Expand Down
10 changes: 8 additions & 2 deletions examples/reference/panes/JPG.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,10 @@
"metadata": {},
"outputs": [],
"source": [
"jpg_pane = pn.pane.JPG('https://www.gstatic.com/webp/gallery/4.sm.jpg')\n",
"jpg_pane = pn.pane.JPG(\n",
" 'https://assets.holoviz.org/panel/samples/jpeg_sample.jpg',\n",
" link_url='https://blog.holoviz.org/panel_0.13.0.html',\n",
" width=500)\n",
"\n",
"jpg_pane"
]
Expand All @@ -61,7 +64,10 @@
"metadata": {},
"outputs": [],
"source": [
"jpg_pane.object = 'https://www.gstatic.com/webp/gallery/1.sm.jpg'"
"jpg_pane.param.update(\n",
" object='https://assets.holoviz.org/panel/samples/jpeg_sample2.jpg',\n",
" link_url='https://blog.holoviz.org/panel_0.14.html'\n",
")"
]
}
],
Expand Down
6 changes: 2 additions & 4 deletions examples/reference/panes/PDF.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,7 @@
"metadata": {},
"outputs": [],
"source": [
"pdf_pane = pn.pane.PDF(\n",
" 'https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf', width=700, height=1024\n",
")\n",
"pdf_pane = pn.pane.PDF('https://assets.holoviz.org/panel/samples/pdf_sample.pdf', width=700, height=1000)\n",
"\n",
"pdf_pane"
]
Expand All @@ -62,7 +60,7 @@
"metadata": {},
"outputs": [],
"source": [
"pdf_pane.object = 'http://www.africau.edu/images/default/sample.pdf'"
"pdf_pane.object = 'https://assets.holoviz.org/panel/samples/pdf_sample2.pdf'"
]
}
],
Expand Down
2 changes: 1 addition & 1 deletion examples/reference/panes/VTKJS.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@
"metadata": {},
"outputs": [],
"source": [
"vtk_pane.object = \"https://github.com/Kitware/vtk-js-datasets/raw/master/data/vtkjs/TBarAssembly.vtkjs\""
"vtk_pane.object = \"https://raw.githubusercontent.com/Kitware/vtk-js-datasets/master/data/vtkjs/TBarAssembly.vtkjs\""
]
},
{
Expand Down
5 changes: 1 addition & 4 deletions examples/reference/panes/Video.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,7 @@
"metadata": {},
"outputs": [],
"source": [
"video = pn.pane.Video(\n",
" 'http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4',\n",
" width=640, height=360, loop=True, time=18\n",
")\n",
"video = pn.pane.Video('https://assets.holoviz.org/panel/samples/video_sample.mp4', width=640, loop=True)\n",
"\n",
"video"
]
Expand Down
2 changes: 2 additions & 0 deletions examples/reference/widgets/TextToSpeech.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"Please note this does not visually display any widget. But you still need to add it to your notebook or server app.\n",
"\n",
"As long as `auto_speak` is true setting the `value` will cause it to speak:"
]
},
Expand Down
2 changes: 1 addition & 1 deletion examples/reference/widgets/VideoStream.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
"\n",
"For layout and styling related parameters see the [customization user guide](../../user_guide/Customization.ipynb).\n",
"\n",
"* **``format``** (str): Format of the captured images, either 'png' or 'jpeg'\n",
"* **``format``** (str): Format of the captured images, either 'png' (default) or 'jpeg'. Choose `jpeg` if you want the `VideoStream` to take high frequent snapshots as the image size is much smaller.\n",
"* **``paused``** (boolean): Whether the video stream is paused\n",
"* **``timeout``** (int): Interval between snapshots (if None then snapshot only taken if snapshot method is called)\n",
"* **``value``** (string): String representation of the current snapshot\n",
Expand Down
106 changes: 77 additions & 29 deletions scripts/generate_panelite_content.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,34 +14,79 @@
EXAMPLES_DIR = PANEL_BASE / 'examples'

# Add piplite command to notebooks
DEPENDENCIES = [repr(d) for d in ['panel', 'pyodide-http', 'altair', 'hvplot', 'matplotlib', 'plotly', 'pydeck', 'scikit-learn']]

nbs = list(EXAMPLES_DIR.glob('*/*/*.ipynb')) + list(EXAMPLES_DIR.glob('*/*.*'))
install = nbformat.v4.new_code_cell(source=f"import piplite\nawait piplite.install([{', '.join(DEPENDENCIES)}])")
del install['id']

for nb in nbs:
nbpath = pathlib.Path(nb)
out = (PANEL_BASE / 'lite/files') / nbpath.relative_to(EXAMPLES_DIR)
out.parent.mkdir(parents=True, exist_ok=True)
if nb.suffix == '.ipynb':
with open(nb, encoding='utf-8') as fin:
nb = nbformat.read(fin, 4)
nb['cells'].insert(0, install)
with open(out, 'w', encoding='utf-8') as fout:
nbformat.write(nb, fout)
elif not nb.is_dir():
shutil.copyfile(nb, out)

# Copy assets
shutil.copytree(
EXAMPLES_DIR / 'assets',
PANEL_BASE / 'lite' / 'files' / 'assets',
dirs_exist_ok=True
)

# Download sampledata
def download():
DEFAULT_DEPENDENCIES = ['panel', 'pyodide-http', 'altair', 'hvplot', 'matplotlib', 'plotly', 'pydeck', 'scikit-learn']
with open(PANEL_BASE/"scripts"/"panelite_dependencies.json", "r", encoding="utf8") as file:
DEPENDENCIES = json.load(file)
DEPENDENCY_NOT_IMPORTABLE = [
"streamz", # https://github.com/python-streamz/streamz/issues/467,
"vtk", # https://gitlab.kitware.com/vtk/vtk/-/issues/18806
]
NOTEBOOK_ISSUES = {
"Getting_Started.ipynb": ["https://github.com/holoviz/panel/issues/4416"],
"reference/panes/DataFrame.ipynb": ["https://github.com/python-streamz/streamz/issues/467"],
"reference/panes/HoloViews.ipynb": ["https://github.com/holoviz/panel/issues/4393"],
"reference/panes/IPyWidget.ipynb": ["https://github.com/holoviz/panel/issues/4394", "https://github.com/widgetti/ipyvolume/issues/427"],
"reference/panes/Matplotlib.ipynb": ["https://github.com/holoviz/panel/issues/4394"],
"reference/panes/Param.ipynb": ["https://github.com/holoviz/panel/issues/4393"],
"reference/panes/Reacton.ipynb": ["https://github.com/holoviz/panel/issues/4394"],
"reference/panes/Str.ipynb": ["https://github.com/holoviz/panel/issues/4396"],
"reference/panes/Streamz.ipynb": ["https://github.com/python-streamz/streamz/issues/467"],
"reference/panes/VTK.ipynb": ["https://gitlab.kitware.com/vtk/vtk/-/issues/18806"],
"reference/panes/VTKVolume.ipynb": ["https://gitlab.kitware.com/vtk/vtk/-/issues/18806"],
"reference/widgets/CrossSelector.ipynb": ["https://github.com/holoviz/panel/issues/4398"],
"reference/widgets/Debugger.ipynb": ["https://github.com/holoviz/panel/issues/4399"],
"reference/widgets/EditableIntSlider.ipynb": ["https://github.com/holoviz/panel/issues/4400"],
"reference/widgets/FileDownload.ipynb": ["https://github.com/holoviz/panel/issues/4401"],
"reference/widgets/IntRangeSlider.ipynb": ["https://github.com/holoviz/panel/issues/4402"],
"reference/widgets/MultiChoice.ipynb": ["https://github.com/holoviz/panel/issues/4403"],
"reference/widgets/RangeSlider.ipynb": ["https://github.com/holoviz/panel/issues/4402"],
"reference/widgets/SpeechToText.ipynb": ["https://github.com/holoviz/panel/issues/4404"],
"reference/widgets/Terminal.ipynb": ["https://github.com/holoviz/panel/issues/4407"],
"gallery/components/VuePdbInput.ipynb": ["https://github.com/holoviz/panel/issues/4417"],
}

def _get_dependencies(nbpath: pathlib.Path):
key = str(nbpath).split("examples/")[-1]
dependencies = DEPENDENCIES.get(key, DEFAULT_DEPENDENCIES)
dependencies = [repr(d) for d in dependencies if not d in DEPENDENCY_NOT_IMPORTABLE]
return dependencies


def _to_source(dependencies):
return f"import piplite\nawait piplite.install([{', '.join(dependencies)}])"


def _get_install_code_cell(nbpath: pathlib.Path):
dependencies = _get_dependencies(nbpath)
source = _to_source(dependencies)
install = nbformat.v4.new_code_cell(source=source)
del install['id']
return install

def copy_examples():
nbs = list(EXAMPLES_DIR.glob('*/*/*.ipynb')) + list(EXAMPLES_DIR.glob('*/*.*'))
for nb in nbs:
nbpath = pathlib.Path(nb)
out = (PANEL_BASE / 'lite/files') / nbpath.relative_to(EXAMPLES_DIR)
out.parent.mkdir(parents=True, exist_ok=True)
if nb.suffix == '.ipynb':
with open(nb, encoding='utf-8') as fin:
nb = nbformat.read(fin, 4)
install = _get_install_code_cell(nbpath)
nb['cells'].insert(0, install)
with open(out, 'w', encoding='utf-8') as fout:
nbformat.write(nb, fout)
elif not nb.is_dir():
shutil.copyfile(nb, out)

def copy_assets():
shutil.copytree(
EXAMPLES_DIR / 'assets',
PANEL_BASE / 'lite' / 'files' / 'assets',
dirs_exist_ok=True
)

def download_sample_data():
"""
Download larger data sets for various Bokeh examples.
"""
Expand Down Expand Up @@ -71,4 +116,7 @@ def download():
print(f"Re-fetching {filename!r} (checksum mismatch)")
_download_file(s3, filename, data_dir, progress=False)

download()
if __name__=="__main__":
copy_examples()
copy_assets()
download_sample_data()
Loading

0 comments on commit f0c64ae

Please sign in to comment.