diff --git a/.github/actions/build-ext/action.yml b/.github/actions/build-ext/action.yml new file mode 100644 index 0000000..300eb4d --- /dev/null +++ b/.github/actions/build-ext/action.yml @@ -0,0 +1,10 @@ +name: "Build" +runs: + using: "composite" + steps: + - name: Build the extension + run: | + set -eux + python -m pip install . + shell: bash + \ No newline at end of file diff --git a/.github/actions/post-test/action.yml b/.github/actions/post-test/action.yml new file mode 100644 index 0000000..d6a1637 --- /dev/null +++ b/.github/actions/post-test/action.yml @@ -0,0 +1,25 @@ +name: "Post Test" +runs: + using: "composite" + steps: + - name: Install check-manifest + run: python -m pip install -U check-manifest + shell: bash + - name: Install Node (for Playwright) + uses: actions/setup-node@v1 + with: + node-version: '16.x' + - name: Install Playwright + run: npx playwright install + shell: bash + - name: extension checks + run: | + jupyter server extension list 2>&1 | grep -ie "rucio_jupyterlab.*OK" + jupyter labextension list 2>&1 | grep -ie "rucio-jupyterlab.*OK" + shell: bash + - name: Browser check + run: python -m jupyterlab.browser_check + shell: bash + - name: Check Manifest.in + run: check-manifest -v + shell: bash \ No newline at end of file diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 0000000..751794f --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,40 @@ +name: "Setup" +runs: + using: "composite" + steps: + - name: Install node + uses: actions/setup-node@v1 + with: + node-version: '14.x' + - name: Install Python + uses: actions/setup-python@v2 + with: + python-version: '3.9' + architecture: 'x64' + - name: Setup pip cache + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: pip-3.9-${{ hashFiles('package.json') }} + restore-keys: | + pip-3.9- + pip- + - name: Get yarn cache directory path + id: yarn-cache-dir-path + run: echo "::set-output name=dir::$(yarn cache dir)" + shell: bash + - name: Setup yarn cache + uses: actions/cache@v2 + id: yarn-cache + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: yarn-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + yarn- + - name: Install Python dependencies + run: python -m pip install -r requirements.txt + shell: bash + - name: Install JS dependencies + run: jlpm + shell: bash + \ No newline at end of file diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml new file mode 100644 index 0000000..c20c932 --- /dev/null +++ b/.github/actions/test/action.yml @@ -0,0 +1,16 @@ +name: "Test" +runs: + using: "composite" + steps: + - name: Run ESLint + run: jlpm run eslint:check + shell: bash + - name: Run Jest + run: jlpm jest + shell: bash + - name: Run Pytest + run: pytest rucio_jupyterlab/tests/ + shell: bash + - name: Run Pylint + run: pylint --fail-under=8.5 rucio_jupyterlab/ + shell: bash diff --git a/.github/workflows/build-and-publish-tagged.yml b/.github/workflows/build-and-publish-tagged.yml index 7eb4dc8..704c873 100644 --- a/.github/workflows/build-and-publish-tagged.yml +++ b/.github/workflows/build-and-publish-tagged.yml @@ -9,61 +9,13 @@ jobs: name: Build and Publish to PyPI runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v1 - - name: Install node - uses: actions/setup-node@v1 - with: - node-version: '14.x' - - name: Install Python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - architecture: 'x64' - - name: Setup pip cache - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: pip-3.8-${{ hashFiles('package.json') }} - restore-keys: | - pip-3.8- - pip- - - name: Get yarn cache directory path - id: yarn-cache-dir-path - run: echo "::set-output name=dir::$(yarn cache dir)" - - name: Setup yarn cache - uses: actions/cache@v2 - id: yarn-cache - with: - path: ${{ steps.yarn-cache-dir-path.outputs.dir }} - key: yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - yarn- - - name: Install Python dependencies + - uses: actions/checkout@v3 + - uses: ./.github/actions/setup + - uses: ./.github/actions/test + - uses: ./.github/actions/build-ext + - uses: ./.github/actions/post-test + - name: Build sdist run: | - python -m pip install -U jupyterlab~=3.1 check-manifest - python -m pip install -r requirements.txt - - name: Install JS dependencies - run: jlpm - - name: Run ESLint - run: jlpm run eslint:check - - name: Run Jest - run: jlpm jest - - name: Run Pytest - run: pytest rucio_jupyterlab/tests/ - - name: Run Pylint - run: pylint --fail-under=8.5 rucio_jupyterlab/ - - name: Build the extension - run: | - set -eux - python -m pip install . - jupyter server extension list 2>&1 | grep -ie "rucio_jupyterlab.*OK" - jupyter labextension list 2>&1 | grep -ie "rucio-jupyterlab.*OK" - python -m pip uninstall jupyter_server_terminals - python -m jupyterlab.browser_check - - check-manifest -v - pip install build python -m build --sdist - name: Publish distribution to PyPI diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a077e4a..7549ed7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,62 +11,9 @@ jobs: build: runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v1 - - name: Install node - uses: actions/setup-node@v1 - with: - node-version: '14.x' - - name: Install Python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - architecture: 'x64' - - name: Setup pip cache - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: pip-3.8-${{ hashFiles('package.json') }} - restore-keys: | - pip-3.8- - pip- - - name: Get yarn cache directory path - id: yarn-cache-dir-path - run: echo "::set-output name=dir::$(yarn cache dir)" - - name: Setup yarn cache - uses: actions/cache@v2 - id: yarn-cache - with: - path: ${{ steps.yarn-cache-dir-path.outputs.dir }} - key: yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - yarn- - - name: Install Python dependencies - run: | - python -m pip install -U jupyterlab~=3.1 check-manifest - python -m pip install -r requirements.txt - - name: Install JS dependencies - run: jlpm - - name: Run ESLint - run: jlpm run eslint:check - - name: Run Jest - run: jlpm jest - - name: Run Pytest - run: pytest rucio_jupyterlab/tests/ - - name: Run Pylint - run: pylint --fail-under=8.5 rucio_jupyterlab/ - - name: Install Node (for Playwright) - uses: actions/setup-node@v1 - with: - node-version: '16.x' - - name: Install Playwright - run: npx playwright install - - name: Build the extension - run: | - set -eux - python -m pip install . - jupyter server extension list 2>&1 | grep -ie "rucio_jupyterlab.*OK" - jupyter labextension list 2>&1 | grep -ie "rucio-jupyterlab.*OK" - python -m jupyterlab.browser_check + - uses: actions/checkout@v3 + - uses: ./.github/actions/setup + - uses: ./.github/actions/test + - uses: ./.github/actions/build-ext + - uses: ./.github/actions/post-test - check-manifest -v diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..34bf610 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "rucio"] + path = docker/test_env/rucio + url = git@github.com:rucio/rucio.git +[submodule "docker/test_env/rucio"] + path = docker/test_env/rucio + url = https://github.com/rucio/rucio.git diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 33576cd..0000000 --- a/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -FROM jupyter/scipy-notebook -LABEL maintainer="Muhammad Aditya Hilmy " - -USER $NB_UID - -RUN conda install -y -c conda-forge python-gfal2 \ - && conda clean --all -f -y - -USER root - -RUN apt update -y \ - && apt install -y voms-clients-java \ - && apt clean -y \ - && rm /opt/conda/bin/voms-proxy-init \ - && ln -s /usr/bin/voms-proxy-init /opt/conda/bin/voms-proxy-init - -COPY . /rucio-jupyterlab -WORKDIR /rucio-jupyterlab - -RUN fix-permissions /rucio-jupyterlab \ - && sed -i -e 's/\r$/\n/' /rucio-jupyterlab/docker/configure.sh - -USER $NB_UID - -RUN pip install -e . \ - && jupyter serverextension enable --py rucio_jupyterlab --sys-prefix \ - && jupyter labextension link . --dev-build=False \ - && jupyter lab clean -y \ - && npm cache clean --force \ - && rm -rf "/home/${NB_USER}/.cache/yarn" \ - && rm -rf "/home/${NB_USER}/.node-gyp" - -ENV JUPYTER_ENABLE_LAB=yes - -WORKDIR $HOME -CMD ["/rucio-jupyterlab/docker/configure.sh", "start-notebook.sh"] \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in index 295cb28..e6af632 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -27,7 +27,7 @@ global-exclude .git global-exclude .ipynb_checkpoints exclude Dockerfile -exclude docker/* +recursive-exclude docker * exclude jest.config.js exclude requirements.txt exclude rucio_jupyterlab/tests/* diff --git a/docker/configure.sh b/docker/configure.sh deleted file mode 100755 index 4d8f9fd..0000000 --- a/docker/configure.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -set -e -python /rucio-jupyterlab/docker/configure.py -exec "$@" \ No newline at end of file diff --git a/docker/container/Dockerfile b/docker/container/Dockerfile new file mode 100644 index 0000000..04b9707 --- /dev/null +++ b/docker/container/Dockerfile @@ -0,0 +1,40 @@ +FROM jupyter/docker-stacks-foundation:python-3.9 +LABEL maintainer="Muhammad Aditya Hilmy , Francesc Torradeflot " + +ENV CONTAINER_PURPOSE="test" +ARG NB_USER="jovyan" +ARG NB_UID="1000" +ARG NB_GID="100" + +USER $NB_UID + +RUN mamba install -y -n base -c conda-forge \ + python-gfal2 \ + nodejs=14 \ + pip \ + && mamba clean --all -f -y + +USER root + +COPY . /rucio-jupyterlab +WORKDIR /rucio-jupyterlab + +RUN fix-permissions /rucio-jupyterlab \ + && sed -i -e 's/\r$/\n/' /rucio-jupyterlab/docker/container/configure.sh + +COPY docker/container/jupyter_server_config.py $HOME/.jupyter/jupyter_server_config.py +RUN fix-permissions $HOME/.jupyter +RUN mkdir -p /opt/rucio/etc \ + && chown -R ${NB_UID}:${NB_GID} /opt/rucio/etc + +USER $NB_UID + +RUN /rucio-jupyterlab/docker/container/install_extension.sh + +ENV JUPYTER_ENABLE_LAB=yes + +WORKDIR $HOME +ENTRYPOINT ["/rucio-jupyterlab/docker/container/docker-entrypoint.sh"] +CMD ["/rucio-jupyterlab/docker/container/configure.sh", "/rucio-jupyterlab/docker/container/start_notebook.sh"] + +USER ${NB_UID} \ No newline at end of file diff --git a/docker/container/README.md b/docker/container/README.md new file mode 100644 index 0000000..bbc8a01 --- /dev/null +++ b/docker/container/README.md @@ -0,0 +1,43 @@ +# Rucio jupyterlab extension docker image + +## Building the image + +### Base image: jupyter/docker-stacks-foundation + +We rely on the `docker-stacks-foundation` image from the +[Jupyter Docker Stacks repository](https://github.com/jupyter/docker-stacks/tree/main). + +It is an image containing a minimal conda/mamba environment, we want to use this image to make +sure the extension works in a "clean" environment. + +### Building a custom base image + +In our case we want to set the python version to 3.9, and there's no image publicly available with this version. + +So we clone the [Jupyter Docker Stacks repository](https://github.com/jupyter/docker-stacks/tree/main) + + git clone git@github.com:jupyter/docker-stacks.git jupyter-docker-stacks + cd jupyter-docker-stacks/images/docker-stacks-foundation/ + +Tune the Dockerfile to install python=3.9, and build/tag the image that will be used as the basis +for our `rucio-jupyterlab` image. + + docker build . -t jupyter/docker-stacks-foundation:python-3.9 + +### Building the `rucio-jupyterlab` docker image + +From the root folder of the project, run + + docker build . -t rucio-jupyterlab -f docker/container/Dockerfile + +## Using the image + +### Standalone + +Check the instructions in [the main README](../../README.md) to see how to use the image +standalone, that is without the rest of the Rucio development environment + +### Inside Rucio's development environment + +Check the instructions on [how to use the test environment](../test_env/README.md) to +see how to use this image together with the Rucio development environment. diff --git a/docker/configure.py b/docker/container/configure.py similarity index 77% rename from docker/configure.py rename to docker/container/configure.py index 10c41bb..e3dc6d3 100644 --- a/docker/configure.py +++ b/docker/container/configure.py @@ -9,6 +9,7 @@ import os import json +import configparser HOME = '/home/jovyan' @@ -63,6 +64,30 @@ def write_jupyterlab_config(): config_file.write(json.dumps(config_json, indent=2)) config_file.close() +def write_rucio_config(): + + rucio_config = configparser.ConfigParser() + + client_config = { + 'rucio_host': os.getenv('RUCIO_BASE_URL'), + 'auth_host': os.getenv('RUCIO_AUTH_URL'), + 'auth_type': os.getenv('RUCIO_AUTH_TYPE', 'userpass'), # it could be gss, x509_proxy, ssh + 'username': os.getenv('RUCIO_USERNAME', 'ddmlab'), + 'password': os.getenv('RUCIO_PASSWORD', 'secret'), + 'ca_cert': os.getenv('RUCIO_CA_CERT'), + 'client_cert': os.getenv('X509_USER_CERT'), + 'client_key': os.getenv('X509_USER_KEY'), + 'client_x509_proxy': os.getenv('X509_USER_PROXY'), + 'ssh_private_key': os.getenv('SSH_PRIVATE_KEY', '$HOME/.ssh/id_rsa'), + 'account': os.getenv('RUCIO_ACCOUNT', 'root'), + 'request_retries': 3, + 'protocol_stat_retries': 6 + } + client_config = dict((k, v) for k, v in client_config.items() if v) + + rucio_config['client'] = client_config + with open('/opt/rucio/etc/rucio.cfg', 'w') as f: + rucio_config.write(f) def write_ipython_config(): file_path = HOME + '/.ipython/profile_default/ipython_kernel_config.json' @@ -99,3 +124,4 @@ def write_ipython_config(): if __name__ == '__main__': write_jupyterlab_config() write_ipython_config() + write_rucio_config() diff --git a/docker/container/configure.sh b/docker/container/configure.sh new file mode 100755 index 0000000..8aa5593 --- /dev/null +++ b/docker/container/configure.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -e +echo "Configuring Rucio jupyterlab extension" +python /rucio-jupyterlab/docker/container/configure.py +exec "$@" \ No newline at end of file diff --git a/docker/container/docker-entrypoint.sh b/docker/container/docker-entrypoint.sh new file mode 100755 index 0000000..26e77f1 --- /dev/null +++ b/docker/container/docker-entrypoint.sh @@ -0,0 +1,17 @@ +#!/bin/bash +#eval "$(micromamba shell hook --shell bash)" +#micromamba activate base + +# Step zero, get a compliant proxy. The key must NOT be group/other readable + +if [[ -f ${X509_USER_KEY} ]] && [[ -f ${X509_USER_CERT} ]] +then + KEY=$(mktemp) + cat ${X509_USER_KEY} > "$KEY" + xrdgsiproxy init -valid 9999:00 -cert ${X509_USER_CERT} -key "$KEY" + rm -f "$KEY" +else + echo "No certificate and/or key provided to create a proxy" +fi + +exec "$@" diff --git a/docker/container/install_extension.sh b/docker/container/install_extension.sh new file mode 100755 index 0000000..bfd5f90 --- /dev/null +++ b/docker/container/install_extension.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# install the rucio-jupyterlab extension +# if the purpose is develop, build it +# otherwise just install it +#eval "$(micromamba shell hook --shell bash )" +#micromamba activate base +cd /rucio-jupyterlab + +if [ "$CONTAINER_PURPOSE" == "develop" ] +then + pip install -e . + jupyter labextension develop --overwrite . + jlpm run build + jupyter server extension enable rucio_jupyterlab.server +else + pip install . + jupyter serverextension enable --py rucio_jupyterlab --sys-prefix + jupyter labextension link . --dev-build=False + jupyter lab clean -y + npm cache clean --force \ + && rm -rf "/home/${NB_USER}/.cache/yarn" \ + && rm -rf "/home/${NB_USER}/.node-gyp" +fi \ No newline at end of file diff --git a/docker/container/jupyter_server_config.py b/docker/container/jupyter_server_config.py new file mode 100644 index 0000000..498843d --- /dev/null +++ b/docker/container/jupyter_server_config.py @@ -0,0 +1,2 @@ +c.ServerApp.token = "" +c.ServerApp.password = "" \ No newline at end of file diff --git a/docker/container/start_notebook.sh b/docker/container/start_notebook.sh new file mode 100755 index 0000000..a4a3019 --- /dev/null +++ b/docker/container/start_notebook.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +jupyter lab --allow-root --ip 0.0.0.0 --no-browser --port 8888 --debug \ No newline at end of file diff --git a/docker/test_env/README.md b/docker/test_env/README.md new file mode 100644 index 0000000..16c1845 --- /dev/null +++ b/docker/test_env/README.md @@ -0,0 +1,131 @@ +# Rucio jupyterlab extension test environment + +## Overview + +This folder contains the necessary files and instructions to launch +a local Rucio cluster with the goal of testing the jupyterlab extension +coupled to specific versions of Rucio and jupyterlab. + +The cluster is an extension of the +[Rucio development cluster](https://github.com/rucio/rucio/tree/master/etc/docker/dev) +with an additional container (`rucio-jupyterlab`) containing a jupyterlab server +with the checked-out version of the Rucio jupyterlab extension. + +## Preparation + +### Build the rucio-jupyterlab docker image + +Check the documentation in the [container folder](../container/README.md) + +### Choose the version of Rucio + +Rucio is added as a git submodule. + +If it's the first time you clone this repo, +you will have to run `git submodule init` to initialize it, and `git submodule update` to fetch +the updates. + +Then you can choose the version of Rucio you want to use by doing a `git checkout`, for example: +``` +cd rucio +git checkout 32.8.0 +``` + + +## Managing the development cluster + +### Start the cluster + +Execute the `run_test_env.sh` script to launch a development cluster. + +This script will also create a temporary folder in `$TMP` (`/tmp/rucio_xrd1`) that will be shared between +the XRD1 RSE and the jupyterlab container to test the extension in `replica` mode. + +The script also runs the tests to create some dummy data. If the temporary folder already exists, +it is assumed that the dummy data was already created and this step is skipped. + +Once the script is executed it will take some time for the rucio-jupyterlab container +to be ready. It has to install all the dependencies, build the extension, ... +The progress of the setup can be tracked through the container logs +`docker logs -t dev-rucio-jupyterlab-1` . When the process finishes there will be +this message in the logs +``` +[I 2024-02-08 12:42:04.428 ServerApp] Jupyter Server 1.24.0 is running at: +[I 2024-02-08 12:42:04.428 ServerApp] http://693c5ea2930d:8888/lab +[I 2024-02-08 12:42:04.428 ServerApp] or http://127.0.0.1:8888/lab +[I 2024-02-08 12:42:04.428 ServerApp] Use Control-C to stop this server and shut down all kernels (twice to skip confirmation). +``` +and the jupyterlab server will be accessible at http://localhost:8888 + + +### Stop the cluster + +Execute the `stop_test_env.sh` script to stop de cluster, +remove the containers and the temporary directory. + + +## Testing the extension + +### Accessing the jupyterlab server + +Once the development cluster is running, the jupyerlab server should be reachable +through a web browser at http://localhost:8888 + +### Check Rucio + +From a terminal in jupyterlab you can check that the Rucio credentials are correct: + + rucio whoami + +Perform other rucio commands at your wish, for example: + + rucio list-dids --filter 'type=all' test:* + +will list all DIDs in the `test` scope. + +When starting the dev cluster, some replication rules will be +created but they will be stuck in `REPLICATING`. +See the section on [How to trigger replicas](#how-to-trigger-replicas) + +### Testing the extension + +* Go to the Rucio jupyterlab extension dashboard no the left-side panel. +* Fill in the Settings: + * Active Instance = "Rucio Test" + * Authentication = "Username & Password" + * Username = "ddmlab" + * Password = "secret" + * Account = "root" +* Go to the **Explore** tab, look for `test:file3` and then click on "Make Available" + * If the extension is working in `download` mode + the file should be available locally after some minutes + * If the extension is working in `replica` mode a replication rule will + be added but the replication process will be stuck because there are no Rucio daemons + in the dev cluster see the section [How to trigger replicas](#how-to-trigger-replicas) +* Once the file is "Available", open a Notebook +* The "Add to Notebook" button should appear next to the "Available" label, +click on it and follow the steps to add an environment variable containing the path to the +DID in the notebook +* A green "Ready" label should be visible in the top bar of the notebook +* The attached DIDs should be listed in the Notebook tab of the Rucio jupyterlab extension dashboard + + +## Troubleshooting + +### How to trigger replicas + +When using the development cluster, the daemons are not deployed so you have to run them +in single execution mode to actually get the replication done. + +Open a terminal inside the `dev-rucio-1` container by running this command +in the host machine (not in jupyterlab) + + docker exec -it dev-rucio-1 /bin/bash + +Then run these commands + + rucio-conveyor-submitter --run-once + rucio-conveyor-poller --run-once --older-than 0 + rucio-conveyor-finisher --run-once + + diff --git a/docker/test_env/docker-compose-extension.yml b/docker/test_env/docker-compose-extension.yml new file mode 100644 index 0000000..938762a --- /dev/null +++ b/docker/test_env/docker-compose-extension.yml @@ -0,0 +1,49 @@ +services: + ruciodb: + extends: + file: docker-compose.yml + service: ruciodb + rucio: + extends: + file: docker-compose.yml + service: rucio + depends_on: + - ruciodb + rucio-jupyterlab: + image: "rucio-jupyterlab" + build: + context: . + profiles: + - extension + volumes: + - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z + - ../../certs/hostcert_rucio.pem:/etc/grid-security/hostcert.pem:z + - ../../certs/hostcert_rucio.key.pem:/etc/grid-security/hostkey.pem:z + - ../../certs/rucio_ca.pem:/opt/rucio/etc/rucio_ca.pem:z + - ../../certs/ruciouser.pem:/opt/rucio/etc/usercert.pem:z + - ../../certs/ruciouser.key.pem:/opt/rucio/etc/userkey.pem:z + - ../../../../../..:/rucio-jupyterlab:Z + - /tmp/rucio_xrd1:/tmp/rucio_xrd1:z + environment: + - X509_USER_CERT=/opt/rucio/etc/usercert.pem + - X509_USER_KEY=/opt/rucio/etc/userkey.pem + - RUCIO_CA_CERT=/opt/rucio/etc/rucio_ca.pem + - RUCIO_MODE=replica + - RUCIO_WILDCARD_ENABLED=1 + - RUCIO_BASE_URL=https://rucio + - RUCIO_AUTH_URL=https://rucio + - RUCIO_DESTINATION_RSE=XRD1 + - RUCIO_DISPLAY_NAME=Rucio Test + - RUCIO_NAME=rucio.test + - RUCIO_PATH_BEGINS_AT=1 + - RUCIO_RSE_MOUNT_PATH=/tmp/rucio_xrd1 + - CONTAINER_PURPOSE=develop + ports: + - "8888:8888" + xrd1: + extends: + file: docker-compose.yml + service: xrd1 + volumes: + - /tmp/rucio_xrd1:/rucio:z + \ No newline at end of file diff --git a/docker/test_env/rucio b/docker/test_env/rucio new file mode 160000 index 0000000..20266ee --- /dev/null +++ b/docker/test_env/rucio @@ -0,0 +1 @@ +Subproject commit 20266eed111d4100d121fd81d5a93dc5f38d31ea diff --git a/docker/test_env/run_test_env.sh b/docker/test_env/run_test_env.sh new file mode 100755 index 0000000..7fc34ef --- /dev/null +++ b/docker/test_env/run_test_env.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Create a shared folder to be used for the XRD1 RSE +# and the rucio-jupyterlab container +if [ ! -d /tmp/rucio_xrd1 ] +then + mkdir -p /tmp/rucio_xrd1 + chmod 777 /tmp/rucio_xrd1 + RUN_TEST=1 +else + RUN_TEST=0 +fi + +# Run Docker Compose with the chosen file and profile +docker compose \ + --file rucio/etc/docker/dev/docker-compose.yml \ + --file docker-compose-extension.yml \ + --profile storage --profile extension up -d + + +# Login into rucio dev container and run some tests, create some RSEs, +# file uploads, file replication using rules +# to have a working set-up of RUCIO server +# WARNING: if the test cluster wasn't properly closed this step will crash +if [ "$RUN_TEST" -eq "1" ] +then + docker exec -it dev-rucio-1 /bin/bash -c "tools/run_tests.sh -ir" +fi diff --git a/docker/test_env/stop_test_env.sh b/docker/test_env/stop_test_env.sh new file mode 100755 index 0000000..daff8a9 --- /dev/null +++ b/docker/test_env/stop_test_env.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +docker_compose_file="rucio/etc/docker/dev/docker-compose.yml" + +# stop the cluster +docker compose \ + --file $docker_compose_file \ + --file docker-compose-extension.yml \ + --profile storage \ + --profile extension \ + stop + +# remove all containers +docker compose \ + --file $docker_compose_file \ + --file docker-compose-extension.yml \ + --profile storage \ + --profile extension \ + rm --force + +sudo rm -rf /tmp/rucio_xrd1 diff --git a/package.json b/package.json index 5e053cb..9645ce5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rucio-jupyterlab", - "version": "0.9.8", + "version": "0.10.0", "description": "JupyterLab extension for integrating Rucio", "keywords": [ "jupyter", diff --git a/setup.py b/setup.py index 1e92c4b..c90a11a 100644 --- a/setup.py +++ b/setup.py @@ -56,12 +56,13 @@ packages=setuptools.find_packages(), install_requires=[ "jupyter_server>=1.6,<2", - "requests~=2.25.0", - "peewee~=3.14.0", - "jsonschema~=3.2.0", - "psutil~=5.8.0", - "rucio-clients>=1.26.0", - "pyjwt" + "requests", + "peewee", + "jsonschema", + "psutil", + "rucio-clients>=32.0", + "pyjwt", + "jupyterlab>=3,<4" ], zip_safe=False, include_package_data=True,