diff --git a/Makefile b/Makefile index 937b4d89e..5953d6f1c 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,7 @@ install-ui: $(call INSTALL_LAB_EXTENSION,notebook-search) $(call INSTALL_LAB_EXTENSION,notebook-search-toolbar) $(call INSTALL_LAB_EXTENSION,core) + $(call INSTALL_LAB_EXTENSION,notebook-containerizer) $(call INSTALL_LAB_EXTENSION,chart-customs) $(call INSTALL_LAB_EXTENSION,components) $(call INSTALL_LAB_EXTENSION,experiment-manager) @@ -51,6 +52,7 @@ link-ui: $(call LINK_LAB_EXTENSION,notebook-search) $(call LINK_LAB_EXTENSION,notebook-search-toolbar) $(call LINK_LAB_EXTENSION,core) + $(call LINK_LAB_EXTENSION,notebook-containerizer) $(call LINK_LAB_EXTENSION,chart-customs) $(call LINK_LAB_EXTENSION,components) $(call LINK_LAB_EXTENSION,experiment-manager) @@ -60,6 +62,7 @@ link-ui: dist-ui: build-frontend mkdir -p dist $(call PACKAGE_LAB_EXTENSION,core) + $(call PACKAGE_LAB_EXTENSION,notebook-containerizer) $(call PACKAGE_LAB_EXTENSION,chart-customs) $(call PACKAGE_LAB_EXTENSION,components) $(call PACKAGE_LAB_EXTENSION,experiment-manager) diff --git a/docker/.condarc b/docker/.condarc new file mode 100644 index 000000000..09c5cc0c2 --- /dev/null +++ b/docker/.condarc @@ -0,0 +1,2 @@ +envs_dirs: + - /home/jovyan/conda-envs/ \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 0bcecea99..32c6e71b7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,27 +1,44 @@ -FROM jupyterhub/k8s-singleuser-sample:1.1.3-n248.h20c9028e +FROM qcdis/miniconda3-naavre AS naavre-build +RUN conda install -c conda-forge conda-pack +COPY environment.yaml . +ADD jupyterlab_vre-0.1.0-py3-none-any.whl /home/jovyan/jupyterlab_vre-0.1.0-py3-none-any.whl + +RUN conda env update -f environment.yaml +RUN conda list +RUN conda-pack -n venv -o /tmp/env.tar && \ + mkdir /venv && cd /venv && tar xf /tmp/env.tar && \ + rm /tmp/env.tar +RUN /venv/bin/conda-unpack + +FROM jupyterhub/k8s-singleuser-sample:1.1.3-n248.h20c9028e AS runtime USER root -RUN apt-get update --allow-releaseinfo-change && apt-get -y install s3fs git +RUN apt-get update --allow-releaseinfo-change && apt-get -y install fuse + +COPY --from=naavre-build /venv/ /venv/ + +ENV PATH=/venv/bin:$PATH +ENV PATH=/home/jovyan/.local/bin:$PATH +RUN source /venv/bin/activate +RUN echo "source /venv/bin/activate" >> ~/.bashrc +SHELL ["/bin/bash", "--login", "-c"] ADD start-jupyter.sh /usr/local/bin/start-jupyter.sh -ADD jupyterlab_vre-0.1.0-py3-none-any.whl /home/jovyan/jupyterlab_vre-0.1.0-py3-none-any.whl +ADD start-jupyter-venv.sh /usr/local/bin/start-jupyter-venv.sh COPY ./repo_utils/ /tmp/repo_utils -RUN chmod ugo+x /usr/local/bin/start-jupyter.sh - -USER $NB_USER +COPY .condarc /tmp/.condarc -RUN conda remove --force -y terminado && \ - python -m pip install --upgrade pip -RUN python3 -m pip install --quiet --no-cache-dir /home/jovyan/jupyterlab_vre-0.1.0-py3-none-any.whl && \ - jupyter serverextension enable --py jupyterlab_vre --user +RUN chmod ugo+x /usr/local/bin/start-jupyter.sh +RUN chmod ugo+x /usr/local/bin/start-jupyter-venv.sh +RUN chown $NB_USER -R /venv/ -RUN pip install jupyterlab-github jupyter-videochat nbgitpuller jupyterlab-lsp python-lsp-server[all] pre-commit ggshield && \ - jupyter serverextension enable --py jupyter_videochat --user && \ - jupyter serverextension enable --py jupyterlab_github --user +USER $NB_USER +RUN jupyter serverextension enable --py jupyterlab_vre --user +RUN jupyter serverextension enable --py jupyter_videochat --user +RUN jupyter serverextension enable --py jupyterlab_github --user RUN jupyter lab build --debug; -RUN conda clean -a -y -CMD ["/usr/local/bin/start-jupyter.sh"] \ No newline at end of file +cmd ["/usr/local/bin/start-jupyter-venv.sh"] \ No newline at end of file diff --git a/docker/Dockerfile-MULTIPLY b/docker/Dockerfile-MULTIPLY index 8948b77b8..276e5159d 100644 --- a/docker/Dockerfile-MULTIPLY +++ b/docker/Dockerfile-MULTIPLY @@ -1,59 +1,53 @@ -FROM jupyterhub/k8s-singleuser-sample:1.1.3-n248.h20c9028e +FROM qcdis/miniconda3-naavre AS naavre-build +RUN conda env export --name venv > naavre-build-environment.yml +RUN cat naavre-build-environment.yml + + +FROM qcdis/miniconda3-multiply AS naavre-pdal-build + +RUN conda install -c conda-forge conda-pack +COPY --from=naavre-build naavre-build-environment.yml naavre-build-environment.yml +COPY laserfarm-environment.yaml . +ADD jupyterlab_vre-0.1.0-py3-none-any.whl /home/jovyan/jupyterlab_vre-0.1.0-py3-none-any.whl +RUN conda install -c conda-forge conda-merge conda-pack +RUN conda-merge naavre-build-environment.yml laserfarm-environment.yaml > merged-environment.yaml +RUN cat merged-environment.yaml +RUN conda env update -f merged-environment.yaml +RUN conda list +RUN conda-pack -n venv -o /tmp/env.tar && \ + mkdir /venv && cd /venv && tar xf /tmp/env.tar && \ + rm /tmp/env.tar +RUN /venv/bin/conda-unpack + +FROM jupyterhub/k8s-singleuser-sample:1.1.3-n248.h20c9028e AS runtime USER root -#===== Add pdalpy dependencies ====== -RUN apt-get update --allow-releaseinfo-change && apt-get -y install gcc g++ s3fs curl git -#==================================== +RUN apt-get update --allow-releaseinfo-change && apt-get -y install fuse -#RUN apt-get -y install libblas3 libblas-dev libboost-all-dev bzip2 ca-certificates libcairo2-dev libcfitsio-dev \ -# libc-ares-dev expat fontconfig libfreetype6-dev gettext giflib-tools glib-networking glib-networking-common \ -# gir1.2-gst-plugins-base-1.0 libhdf4-0-alt libhdf5-dev libicu-dev libpixman-1-0 pcre2-utils nodejs lz4 \ -# libopenblas-dev openssl poppler-utils poppler-data postgresql postgresql-contrib \ -# libpthread-stubs0-dev sqlite3 python-tk zlib1g-dev libblas-dev libcurl4 libffi-dev \ -# php-iconv libkmldom1 liblapacke llvm libnetcdf-dev krb5-user +COPY --from=naavre-pdal-build /venv/ /venv/ + +ENV PATH=/venv/bin:$PATH +ENV PATH=/home/jovyan/.local/bin:$PATH +RUN source /venv/bin/activate +RUN echo "source /venv/bin/activate" >> ~/.bashrc +SHELL ["/bin/bash", "--login", "-c"] ADD start-jupyter.sh /usr/local/bin/start-jupyter.sh -ADD jupyterlab_vre-0.1.0-py3-none-any.whl /home/jovyan/jupyterlab_vre-0.1.0-py3-none-any.whl +ADD start-jupyter-venv.sh /usr/local/bin/start-jupyter-venv.sh COPY ./repo_utils/ /tmp/repo_utils +COPY .condarc /tmp/.condarc + + RUN chmod ugo+x /usr/local/bin/start-jupyter.sh +RUN chmod ugo+x /usr/local/bin/start-jupyter-venv.sh +RUN chown $NB_USER -R /venv/ USER $NB_USER +RUN jupyter serverextension enable --py jupyterlab_vre --user +RUN jupyter serverextension enable --py jupyter_videochat --user +RUN jupyter serverextension enable --py jupyterlab_github --user -RUN conda remove --force -y terminado && \ - python -m pip install --upgrade pip - -RUN python3 -m pip install --quiet --no-cache-dir /home/jovyan/jupyterlab_vre-0.1.0-py3-none-any.whl && \ - jupyter serverextension enable --py jupyterlab_vre --user - - -#===== Add pdalpy dependencies ====== -RUN conda install --yes -c conda-forge pdal python-pdal gdal rclone && \ - conda upgrade --yes numpy && \ - conda update --yes -n base conda && \ - pip install dask distributed lazperf==1.4.4 && \ - pip install laserfarm laspy -#==================================== - -RUN pip install jupyterlab-github jupyter-videochat nbgitpuller jupyterlab-lsp python-lsp-server[all] pre-commit ggshield && \ - jupyter serverextension enable --py jupyter_videochat --user && \ - jupyter serverextension enable --py jupyterlab_github --user - -COPY MULTIPLY_pip_requirements.txt requirements.txt -RUN pip install -r requirements.txt -RUN pip install git+https://github.com/JorisTimmermans/data-access.git \ - git+https://github.com/JorisTimmermans/atmospheric_correction.git \ - git+https://github.com/QCDIS/multiply-core.git \ - git+https://github.com/QCDIS/inference-engine.git \ - git+https://github.com/QCDIS/prior-engine.git \ - git+https://github.com/JorisTimmermans/atmospheric_correction.git \ - git+https://github.com/QCDIS/vm-support.git \ - git+https://github.com/multiply-org/multiply-orchestration.git \ - git+https://github.com/QCDIS/KaFKA-InferenceEngine.git \ - git+https://github.com/QCDIS/BRDF_descriptors.git - -WORKDIR $HOME RUN jupyter lab build --debug; -RUN conda clean -a -y -CMD ["/usr/local/bin/start-jupyter.sh"] \ No newline at end of file +cmd ["/usr/local/bin/start-jupyter-venv.sh"] \ No newline at end of file diff --git a/docker/Dockerfile-laserfarm-pytorch b/docker/Dockerfile-laserfarm-pytorch index eac327feb..a42f0d2d9 100644 --- a/docker/Dockerfile-laserfarm-pytorch +++ b/docker/Dockerfile-laserfarm-pytorch @@ -1,40 +1,52 @@ -FROM jupyterhub/k8s-singleuser-sample:1.1.3-n248.h20c9028e +FROM qcdis/miniconda3-naavre AS naavre-build -USER root +RUN conda env export --name venv > naavre-build-environment.yml +RUN cat naavre-build-environment.yml -#===== Add pdalpy dependencies ====== -RUN apt-get update --allow-releaseinfo-change && apt-get -y install gcc g++ git -#==================================== -RUN apt autoclean -y && apt autoremove -y +FROM qcdis/miniconda3-pdal:v0.5 AS naavre-pdal-build -ADD start-jupyter.sh /usr/local/bin/start-jupyter.sh +RUN conda install -c conda-forge conda-pack +COPY --from=naavre-build naavre-build-environment.yml naavre-build-environment.yml +COPY laserfarm-environment.yaml . ADD jupyterlab_vre-0.1.0-py3-none-any.whl /home/jovyan/jupyterlab_vre-0.1.0-py3-none-any.whl -COPY ./repo_utils/ /tmp/repo_utils +RUN conda install -c conda-forge conda-merge conda-pack +RUN conda-merge naavre-build-environment.yml laserfarm-environment.yaml > merged-environment.yaml +RUN cat merged-environment.yaml +RUN conda env update -f merged-environment.yaml +RUN conda list +RUN conda-pack -n venv -o /tmp/env.tar && \ + mkdir /venv && cd /venv && tar xf /tmp/env.tar && \ + rm /tmp/env.tar +RUN /venv/bin/conda-unpack + +FROM jupyterhub/k8s-singleuser-sample:1.1.3-n248.h20c9028e AS runtime +USER root -RUN chmod ugo+x /usr/local/bin/start-jupyter.sh +RUN apt-get update --allow-releaseinfo-change && apt-get -y install fuse -USER $NB_USER +COPY --from=naavre-pdal-build /venv/ /venv/ -RUN conda remove --force -y terminado && \ - python -m pip install --upgrade pip +ENV PATH=/venv/bin:$PATH +ENV PATH=/home/jovyan/.local/bin:$PATH +RUN source /venv/bin/activate +RUN echo "source /venv/bin/activate" >> ~/.bashrc +SHELL ["/bin/bash", "--login", "-c"] -RUN python -m pip install --quiet --no-cache-dir /home/jovyan/jupyterlab_vre-0.1.0-py3-none-any.whl && \ - jupyter serverextension enable --py jupyterlab_vre --user +ADD start-jupyter.sh /usr/local/bin/start-jupyter.sh +ADD start-jupyter-venv.sh /usr/local/bin/start-jupyter-venv.sh +COPY ./repo_utils/ /tmp/repo_utils +COPY .condarc /tmp/.condarc -#===== Add pdalpy dependencies ====== -RUN conda install --yes -c conda-forge pdal python-pdal gdal rclone && \ - conda upgrade --yes numpy && \ - conda update --yes -n base conda && \ - pip install dask distributed lazperf==1.4.4 && \ - pip install laserfarm laspy opencv-python-headless torch torchvision flwr scikit-learn tensorflow -#==================================== +RUN chmod ugo+x /usr/local/bin/start-jupyter.sh +RUN chmod ugo+x /usr/local/bin/start-jupyter-venv.sh +RUN chown $NB_USER -R /venv/ -RUN pip install jupyterlab-github jupyter-videochat matplotlib folium cartopy icoscp pangaeapy nbgitpuller jupyterlab-lsp python-lsp-server[all] pre-commit ggshield && \ - jupyter serverextension enable --py jupyter_videochat --user && \ - jupyter serverextension enable --py jupyterlab_github --user +USER $NB_USER +RUN jupyter serverextension enable --py jupyterlab_vre --user +RUN jupyter serverextension enable --py jupyter_videochat --user +RUN jupyter serverextension enable --py jupyterlab_github --user RUN jupyter lab build --debug; -RUN conda clean -a -y -CMD ["/usr/local/bin/start-jupyter.sh"] \ No newline at end of file +cmd ["/usr/local/bin/start-jupyter-venv.sh"] \ No newline at end of file diff --git a/docker/Dockerfile-vol2bird b/docker/Dockerfile-vol2bird index 5e383b926..523eac596 100644 --- a/docker/Dockerfile-vol2bird +++ b/docker/Dockerfile-vol2bird @@ -1,102 +1,69 @@ +FROM qcdis/miniconda3-naavre:v0.6 AS naavre-build + +RUN conda env export --name venv > naavre-build-environment.yml +RUN cat naavre-build-environment.yml + + + +FROM qcdis/python-vol2birds AS vol2bird + +RUN conda install -c conda-forge conda-pack +COPY --from=naavre-build naavre-build-environment.yml naavre-build-environment.yml +COPY laserfarm-environment.yaml . +ADD jupyterlab_vre-0.1.0-py3-none-any.whl /home/jovyan/jupyterlab_vre-0.1.0-py3-none-any.whl +RUN conda install -c conda-forge conda-merge conda-pack +RUN conda-merge naavre-build-environment.yml laserfarm-environment.yaml > merged-environment.yaml +RUN cat merged-environment.yaml + +RUN conda env update -f merged-environment.yaml +RUN conda list +RUN conda-pack -n venv -o /tmp/env.tar && \ + mkdir /venv && cd /venv && tar xf /tmp/env.tar && \ + rm /tmp/env.tar +RUN /venv/bin/conda-unpack + + FROM jupyterhub/k8s-singleuser-sample:1.1.3-n248.h20c9028e USER root -#===== Add pdalpy dependencies ====== -RUN apt-get update --allow-releaseinfo-change && apt-get upgrade -y && apt-get -y install gcc g++ s3fs curl git -#==================================== - -#============== install vol2bird============== -# installs using apt-get: -# * libconfuse: library for parsing options -# * libhdf5: HDF5, Hierarchichal Data Format library -# * libgsl: the GNU Scientific Library -# * git, for fetching repositories from Github -# * git-lfs, for fetching large file repositories from Github -# * wget for downloading files, specifically libtorch -# * unzip -# * compiler (gcc, g++, make, cmake, etc) -# * zlib (gzip archiving library) -# * libbz2 (bzip2 archiving library) -# * python -# * numpy -# * proj4 library -# * flexold, otherwise configure script of RSL library does not function properly -RUN apt-get update && apt-get install --no-install-recommends -y libconfuse-dev \ - libhdf5-dev gcc g++ wget unzip make cmake zlib1g-dev python-dev python-numpy libproj-dev flex-old file \ - && apt-get install -y git git-lfs && apt-get install -y libgsl-dev && apt-get install -y libbz2-dev bison byacc - - -# get a copy of hlhdf: -# configure and build hlhdf -# strange Docker conflict when attempting to install in /opt/radar/hlhdf, therefore in root radar instead -RUN pip3 install numpy minio==7.0.3 h5py pandas -RUN git clone https://github.com/adokter/hlhdf.git \ - && cd hlhdf && ./configure --prefix=/opt/radar --with-hdf5=/usr/include/hdf5/serial,/usr/lib/x86_64-linux-gnu/hdf5/serial \ - && make && make install && cd .. && rm -rf hlhdf - -# get a copy of rave: -# cd into rave source directory and configure -# using a clone from git://git.baltrad.eu/rave.git \ -RUN git clone https://github.com/adokter/rave.git \ - && cd rave && ./configure --prefix=/opt/radar/rave --with-hlhdf=/opt/radar \ - && make && make install && cd .. && rm -rf rave - -# get a copy of iris2odim: -RUN git clone https://github.com/adokter/iris2odim.git \ - && cd iris2odim && export RAVEROOT=/opt/radar \ - && make && make install && cd .. && rm -rf iris2odim - -# get a copy of RSL: -RUN git clone https://github.com/adokter/rsl.git && cd rsl \ - && ./configure --prefix=/opt/radar/rsl \ - && make AUTOCONF=: AUTOHEADER=: AUTOMAKE=: ACLOCAL=: \ - && make install AUTOCONF=: AUTOHEADER=: AUTOMAKE=: ACLOCAL=: \ - && cd .. && rm -rf rsl - -# get a copy of vol2bird -# configure vol2bird -RUN git clone https://github.com/adokter/vol2bird.git \ - && cd vol2bird && ./configure --prefix=/opt/radar/vol2bird --with-rave=/opt/radar/rave --with-rsl=/opt/radar/rsl \ - --with-gsl=/usr/include/gsl,/usr/lib/x86_64-linux-gnu \ - && make && make install && cd .. && rm -rf vol2bird - -# clean up -RUN apt-get remove -y git git-lfs gcc g++ wget unzip make cmake python-numpy -y python-dev flex-old \ - && apt-get clean && apt -y autoremove && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -# prepare mount points -RUN mkdir data +RUN apt-get update --allow-releaseinfo-change && apt-get -y install fuse + +COPY --from=vol2bird /venv /venv + +COPY --from=vol2bird /opt/radar/ /opt/radar/ +COPY --from=vol2bird /usr/lib/x86_64-linux-gnu /usr/lib/x86_64-linux-gnu + # set the paths to installed libraries and executables ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/radar/lib:/opt/radar/rave/lib:/opt/radar/rsl/lib:/opt/radar/vol2bird/lib:/usr/lib/x86_64-linux-gnu ENV PATH=${PATH}:/opt/radar/vol2bird/bin:/opt/radar/rsl/bin -#=============================== + +ENV PATH=/venv/bin:$PATH +ENV PATH=/home/jovyan/.local/bin:$PATH +RUN source /venv/bin/activate +RUN echo "source /venv/bin/activate" >> ~/.bashrc +SHELL ["/bin/bash", "--login", "-c"] ADD start-jupyter.sh /usr/local/bin/start-jupyter.sh -ADD jupyterlab_vre-0.1.0-py3-none-any.whl /home/jovyan/jupyterlab_vre-0.1.0-py3-none-any.whl +ADD start-jupyter-venv.sh /usr/local/bin/start-jupyter-venv.sh COPY ./repo_utils/ /tmp/repo_utils -RUN chmod ugo+x /usr/local/bin/start-jupyter.sh - -RUN apt-get update && apt-get install -y git +COPY .condarc /tmp/.condarc -USER $NB_USER -RUN conda remove --force -y terminado && \ - python -m pip install --upgrade pip - -RUN conda install --yes -c conda-forge rclone - -RUN python3 -m pip install --quiet --no-cache-dir /home/jovyan/jupyterlab_vre-0.1.0-py3-none-any.whl && \ - jupyter serverextension enable --py jupyterlab_vre --user +RUN chmod ugo+x /usr/local/bin/start-jupyter.sh +RUN chmod ugo+x /usr/local/bin/start-jupyter-venv.sh +RUN chown $NB_USER -R /venv/ -RUN pip install jupyterlab-github jupyter-videochat nbgitpuller jupyterlab-lsp python-lsp-server[all] pre-commit ggshield && \ - jupyter serverextension enable --py jupyter_videochat --user && \ - jupyter serverextension enable --py jupyterlab_github --user +USER $NB_USER +RUN jupyter serverextension enable --py jupyterlab_vre --user +RUN jupyter serverextension enable --py jupyter_videochat --user +RUN jupyter serverextension enable --py jupyterlab_github --user +RUN jupyter nbextension enable --py widgetsnbextension -WORKDIR $HOME RUN jupyter lab build --debug; -RUN conda clean -a -y -CMD ["/usr/local/bin/start-jupyter.sh"] \ No newline at end of file + +cmd ["/usr/local/bin/start-jupyter-venv.sh"] +cmd ["/usr/local/bin/start-jupyter-venv.sh"] \ No newline at end of file diff --git a/docker/MULTIPLY_pip_requirements.txt b/docker/MULTIPLY_pip_requirements.txt index c6e1098a1..1ee37e74a 100644 --- a/docker/MULTIPLY_pip_requirements.txt +++ b/docker/MULTIPLY_pip_requirements.txt @@ -106,7 +106,7 @@ toolz==0.11.2 #==pyhd8ed1ab_0 traittypes==0.2.1 #==pyh9f0ad1d_2 urllib3==1.26.9 #==pyhd8ed1ab_0 werkzeug==2.1.2 #==pyhd8ed1ab_1 -wheel>=0.35.1 #==0.37.1 #==pyhd8ed1ab_0 +wheel>=0.37.1 #==0.37.0 #==pyhd8ed1ab_0 xarray==2022.3.0 #==pyhd8ed1ab_0 zipp==3.8.0 #==pyhd8ed1ab_0 zstd==1.5.2 #==h8a70e8d_1 diff --git a/docker/environment.yaml b/docker/environment.yaml new file mode 100644 index 000000000..18c133286 --- /dev/null +++ b/docker/environment.yaml @@ -0,0 +1,10 @@ +name: venv +channels: + - conda-forge + - defaults +dependencies: + - pip + - rclone + - jupyterhub + - pip: + - /home/jovyan/jupyterlab_vre-0.1.0-py3-none-any.whl \ No newline at end of file diff --git a/docker/install_MULTIPLY.sh b/docker/install_MULTIPLY.sh deleted file mode 100644 index 831d6892c..000000000 --- a/docker/install_MULTIPLY.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/sh - -mkdir dependencies -cd dependencies - -# -pip install sentinelhub --upgrade - -git clone https://github.com/JorisTimmermans/atmospheric_correction.git -cd atmospheric_correction -python setup.py develop -cd ../ - - -git clone https://github.com/JorisTimmermans/BRDF_descriptors.git -BRDF_descriptors -python setup.py install -cd ../ - -git clone https://github.com/JorisTimmermans/data-access.git -cd data-access -python setup.py develop -cd ../ - - -git clone https://github.com/multiply-org/inference-engine.git -cd inference-engine -python setup.py develop -cd ../ - -git clone https://github.com/JorisTimmermans/multiply-core.git -cd multiply-core -python setup.py develop -cd ../ - -git clone https://github.com/multiply-org/KaFKA-InferenceEngine.git -cd KaFKA-InferenceEngine -python setup.py develop -cd ../ - -git clone https://github.com/multiply-org/sar-pre-processing.git -cd sar-pre-processing -python setup.py develop -cd ../ - -git clone https://github.com/JorisTimmermans/vm-support.git -cd vm-support -python setup.py develop -cd ../ - diff --git a/docker/laserfarm-environment.yaml b/docker/laserfarm-environment.yaml index 842c21c77..368952227 100644 --- a/docker/laserfarm-environment.yaml +++ b/docker/laserfarm-environment.yaml @@ -2,30 +2,15 @@ name: venv channels: - conda-forge - defaults + - mathieu dependencies: - - python>=3.7 - - pdal - - python-pdal - - gdal - - pip + - pip=22.3.1=pyhd8ed1ab_0 + - ipywidgets + - widgetsnbextension + - dask + - laspy + - jupyterhub + - lazrs-python = 0.4.5 #laserfarm has lazrs 0.4.5 - pip: - - laserfarm - - numpy - - dask - - distributed - - lazperf==1.4.4 - - laserfarm - - laspy - - opencv-python-headless - - torch - - torchvision - - flwr - - scikit-learn - - tensorflow - - matplotlib - - folium - - cartopy - - icoscp - - pangaeapy - - jupyterlab-github - - jupyter-videochat \ No newline at end of file + - laserfarm # TODO enable https://z2jh.jupyter.org/en/stable/jupyterhub/customizing/user-environment.html?highlight=conda#allow-users-to-create-their-own-conda-environments-for-notebooks + - /home/jovyan/jupyterlab_vre-0.1.0-py3-none-any.whl \ No newline at end of file diff --git a/docker/start-jupyter-venv.sh b/docker/start-jupyter-venv.sh new file mode 100644 index 000000000..0b6f52bc4 --- /dev/null +++ b/docker/start-jupyter-venv.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +source /venv/bin/activate +set -e + +# set default ip to 0.0.0.0 +if [[ "${NOTEBOOK_ARGS} $*" != *"--ip="* ]]; then + NOTEBOOK_ARGS="--ip=0.0.0.0 ${NOTEBOOK_ARGS}" +fi + +# shellcheck disable=SC1091,SC2086 +. /usr/local/bin/start.sh jupyterhub-singleuser ${NOTEBOOK_ARGS} "$@" \ No newline at end of file diff --git a/docker/start-jupyter.sh b/docker/start-jupyter.sh index 9129a77cf..b9ba65c6e 100644 --- a/docker/start-jupyter.sh +++ b/docker/start-jupyter.sh @@ -1,6 +1,3 @@ - - - export NB_IP=${NB_IP:-0.0.0.0} export NB_PORT=${NB_PORT:-8888} export JUPYTER_ENABLE_LAB=true diff --git a/environment.yml b/environment.yml index 5067a5f63..f45c08a9d 100644 --- a/environment.yml +++ b/environment.yml @@ -2,12 +2,24 @@ name: jupyterlab channels: - conda-forge - defaults + - mathieu dependencies: - - jupyterlab==3.5.0 - - nodejs==15.14.0 - - python==3.9.2 + - jupyterlab>=3.5.0 + - nodejs>=15.14.0 + - python>=3.8 - yarn + - tinydb>=4.6.1 + - pygithub>=1.55 + - tornado>=6.1 + - notebook + - pyyaml>=6.0 + - nbformat + - autopep8 + - Jinja2 + - distro + - requests + - colorhash - pip - pip: - pre-commit - - ggshield + - ggshield \ No newline at end of file diff --git a/jupyterlab_vre/component_containerizer/handlers.py b/jupyterlab_vre/component_containerizer/handlers.py index f1c465b2c..eeb5453d4 100644 --- a/jupyterlab_vre/component_containerizer/handlers.py +++ b/jupyterlab_vre/component_containerizer/handlers.py @@ -334,13 +334,11 @@ def load_module_names_mapping(): return loaded_module_name_mapping -def build_templates(cell=None, files_info=None): - logger.debug('files_info: ' + str(files_info)) +def map_dependencies(dependencies=None): module_name_mapping = load_module_names_mapping() set_conda_deps = set([]) set_pip_deps = set([]) - logger.debug('cell.dependencies: '+str(cell.dependencies)) - for dep in cell.dependencies: + for dep in dependencies: if 'module' in dep and dep['module']: if '.' in dep['module']: module_name = dep['module'].split('.')[0] @@ -353,15 +351,26 @@ def build_templates(cell=None, files_info=None): pip_package = False if module_name in module_name_mapping['conda'].keys(): module_name = module_name_mapping['conda'][module_name] + pip_package = False conda_package = True if module_name in module_name_mapping['pip'].keys(): module_name = module_name_mapping['pip'][module_name] - pip_package = False + pip_package = True + conda_package = False if not is_standard_module(module_name): if conda_package: set_conda_deps.add(module_name) if pip_package: set_pip_deps.add(module_name) + return set_conda_deps, set_pip_deps + + +def build_templates(cell=None, files_info=None): + logger.debug('files_info: ' + str(files_info)) + + logger.debug('cell.dependencies: '+str(cell.dependencies)) + print('cell.dependencies: ' + str(cell.dependencies)) + set_conda_deps, set_pip_deps = map_dependencies(dependencies=cell.dependencies) loader = PackageLoader('jupyterlab_vre', 'templates') template_env = Environment( loader=loader, trim_blocks=True, lstrip_blocks=True) diff --git a/jupyterlab_vre/tests/test_handlers.py b/jupyterlab_vre/tests/test_handlers.py index 8f6918470..263bfb9ea 100644 --- a/jupyterlab_vre/tests/test_handlers.py +++ b/jupyterlab_vre/tests/test_handlers.py @@ -3,11 +3,13 @@ import os from unittest import mock +from jinja2 import PackageLoader, Environment from tornado.escape import to_unicode from tornado.testing import AsyncHTTPTestCase from tornado.web import Application from jupyterlab_vre import ExtractorHandler, TypesHandler, CellsHandler, ExportWorkflowHandler, ExecuteWorkflowHandler +from jupyterlab_vre.component_containerizer.handlers import map_dependencies from jupyterlab_vre.database.cell import Cell from jupyterlab_vre.database.database import Catalog from jupyterlab_vre.handlers import load_module_names_mapping @@ -45,7 +47,24 @@ def test_export_workflow_handler(self): workflow_path = os.path.join(base_path, 'workflows/splitter.json') with open(workflow_path, 'r') as read_file: payload = json.load(read_file) - response = self.fetch('/exportworkflowhandler', method='POST', body=json.dumps(payload)) + # response = self.fetch('/exportworkflowhandler', method='POST', body=json.dumps(payload)) def test_load_module_names_mapping(self): - load_module_names_mapping() \ No newline at end of file + load_module_names_mapping() + + def test_map_dependencies(self): + dependencies = [{'name': 'pathlib', 'asname': None, 'module': ''}, + {'name': 'numpy', 'asname': 'np', 'module': ''}, + {'name': 'laspy', 'asname': None, 'module': ''}, + {'name': 'Client', 'asname': None, 'module': 'webdav3.client'}, + {'name': 'os', 'asname': None, 'module': ''}, + {'name': 'get_wdclient', 'asname': None, 'module': 'laserfarm.remote_utils'}, + {'name': 'pathlib', 'asname': None, 'module': ''}, + {'name': 'list_remote', 'asname': None, 'module': 'laserfarm.remote_utils'}] + set_conda_deps, set_pip_deps = map_dependencies(dependencies=dependencies) + loader = PackageLoader('jupyterlab_vre', 'templates') + template_env = Environment( + loader=loader, trim_blocks=True, lstrip_blocks=True) + template_conda = template_env.get_template('conda_env_template.jinja2') + template_conda.stream(base_image='cell.base_image', conda_deps=list(set_conda_deps), + pip_deps=list(set_pip_deps)).dump('test_env.yaml') \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index ba6090876..9714d5ecc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,15 @@ -jupyterlab==3.5.0 -wheel==0.37.0 -setuptools_rust==1.1.2 -pyflakes==2.4.0 -colorhash==1.0.4 -requests>=2.27.0 -azure-storage-blob==12.9.0 -tinydb~=4.6.1 -PyGithub~=1.55 -tornado~=6.1 -notebook~=6.4.8 -PyYAML~=6.0 -nbformat~=5.7.0 -autopep8~=1.6.0 -Jinja2~=3.0.3 -setuptools>=65.3.0 -distro~=1.7.0 \ No newline at end of file +jupyterlab>=3.5.0 +setuptools_rust>=1.1.2 +pyflakes>=2.4.0 +colorhash>=1.0.4 +requests>=2.26.0 +tinydb>=4.6.1 +PyGithub>=1.55 +tornado>=6.1 +notebook>=6.4.8 +PyYAML>=6.0 +nbformat>=5.7.0 +autopep8>=1.6.0 +Jinja2>=3.0.3 +setuptools>=63.1.0 +distro>=1.7.0 \ No newline at end of file diff --git a/setup.py b/setup.py index 903c62793..93a525bf2 100644 --- a/setup.py +++ b/setup.py @@ -27,25 +27,21 @@ data_files=[('etc/jupyter/jupyter_server_config.d', glob(jupyter_config_path)), ('share/jupyter/lab/extensions', glob(frontend_packages_path))], install_requires=[ - "jupyterlab~=3.5.0", + "jupyterlab>=3.5.0", "autopep8", - "pyflakes", - "nbformat", - "jinja2", - "colorhash", - "tinydb", - "pyyaml", - "azure-storage-blob", - "PyGithub~=1.55", - "wheel==0.37.0", - "setuptools_rust==1.1.2", - "colorhash==1.0.4", - "requests>=2.27.0", - "tornado~=6.1", - "notebook~=6.4.8", - "PyYAML~=6.0", - "setuptools>=65.3.0", - "distro~=1.7.0" + "pyflakes>=2.4.0", + "nbformat>=5.7.0", + "Jinja2", + "colorhash>=1.0.4", + "tinydb>=4.6.1", + "PyYAML>=6.0", + "PyGithub>=1.55", + "setuptools_rust>=1.5.2", + "requests", + "tornado>=6.1", + "notebook", + "setuptools>=63.1.0", + "distro" ], zip_safe=False, include_package_data=True, diff --git a/test-requirements.txt b/test-requirements.txt index 777cfcbb3..3cb46b920 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,18 +1,17 @@ -tornado-httpclient-mock==0.2.3 -jupyterlab==3.5.0 -wheel==0.37.0 -setuptools_rust==1.1.2 -pyflakes==2.4.0 -colorhash==1.0.4 -requests==2.27.1 -azure-storage-blob==12.9.0 -tinydb~=4.6.1 -PyGithub~=1.55 -tornado~=6.1 -notebook~=6.4.8 -PyYAML~=6.0 -nbformat~=5.7.0 -autopep8~=1.6.0 -Jinja2~=3.0.3 -setuptools>=65.3.0 -distro~=1.7.0 \ No newline at end of file +tornado-httpclient-mock>=0.2.3 +jupyterlab>=3.5.0 +wheel>=0.37.1 +setuptools_rust>=1.1.2 +pyflakes>=2.4.0 +colorhash>=1.0.4 +requests>=2.26.0 +tinydb>=4.6.1 +PyGithub>=1.55 +tornado>=6.1 +notebook>=6.4.8 +PyYAML>=6.0 +nbformat>=5.7.0 +autopep8>=1.6.0 +Jinja2>=3.0.3 +setuptools>=63.1.0 +distro>=1.7.0 \ No newline at end of file