From 55ba632f156c6c12d88d23796dbfc3e918726371 Mon Sep 17 00:00:00 2001 From: Joseph Burling Date: Fri, 10 Jun 2022 14:38:20 -0500 Subject: [PATCH] build(retrocookie): reflect updates to source template from generated project 'science-institute_brain-lab' --- .gitignore | 2 +- .../.github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .../.github/workflows/package-updates.yml | 11 +- .../{{cookiecutter.github_repo}}/README.md | 43 +++++- .../configs/axon.yaml | 9 -- .../docker/.datajoint_config.json | 17 --- .../docker/.devcontainer/.env | 6 + .../docker/.devcontainer/devcontainer.json | 12 +- .../docker-compose.yml} | 43 +++--- .../docker/.gitignore | 7 +- .../docker/README.md | 120 +++++++++-------- .../docker/apt_requirements.txt | 9 -- .../docker/codebook.Dockerfile | 48 ------- .../docker/devcontainer.Dockerfile | 124 ------------------ .../docker/docker-compose-codebook_env.yaml | 33 ----- .../docker-compose-standard_workflow.yaml | 27 ---- .../docker/nofile.txt | 1 - .../docker/standard_worker.Dockerfile | 16 --- .../docker/standard_worker/build.env | 6 + .../docker/standard_worker/run.example.env | 0 20 files changed, 154 insertions(+), 382 deletions(-) delete mode 100644 datajoint-workflow/{{cookiecutter.github_repo}}/configs/axon.yaml delete mode 100644 datajoint-workflow/{{cookiecutter.github_repo}}/docker/.datajoint_config.json create mode 100644 datajoint-workflow/{{cookiecutter.github_repo}}/docker/.devcontainer/.env rename datajoint-workflow/{{cookiecutter.github_repo}}/docker/{docker-compose-devcontainer_env.yaml => .devcontainer/docker-compose.yml} (51%) delete mode 100644 datajoint-workflow/{{cookiecutter.github_repo}}/docker/apt_requirements.txt delete mode 100644 datajoint-workflow/{{cookiecutter.github_repo}}/docker/codebook.Dockerfile delete mode 100644 datajoint-workflow/{{cookiecutter.github_repo}}/docker/devcontainer.Dockerfile delete mode 100644 datajoint-workflow/{{cookiecutter.github_repo}}/docker/docker-compose-codebook_env.yaml delete mode 100644 datajoint-workflow/{{cookiecutter.github_repo}}/docker/docker-compose-standard_workflow.yaml delete mode 100644 datajoint-workflow/{{cookiecutter.github_repo}}/docker/nofile.txt delete mode 100644 datajoint-workflow/{{cookiecutter.github_repo}}/docker/standard_worker.Dockerfile create mode 100644 datajoint-workflow/{{cookiecutter.github_repo}}/docker/standard_worker/build.env create mode 100644 datajoint-workflow/{{cookiecutter.github_repo}}/docker/standard_worker/run.example.env diff --git a/.gitignore b/.gitignore index 422ad00..f7566a0 100644 --- a/.gitignore +++ b/.gitignore @@ -132,4 +132,4 @@ dmypy.json .build/ .DS_Store /build -/science-institute_neuro-lab +/science-institute_brain-lab diff --git a/datajoint-workflow/{{cookiecutter.github_repo}}/.github/ISSUE_TEMPLATE/bug_report.yml b/datajoint-workflow/{{cookiecutter.github_repo}}/.github/ISSUE_TEMPLATE/bug_report.yml index af74827..16fca81 100644 --- a/datajoint-workflow/{{cookiecutter.github_repo}}/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/datajoint-workflow/{{cookiecutter.github_repo}}/.github/ISSUE_TEMPLATE/bug_report.yml @@ -26,7 +26,7 @@ body: attributes: label: Version description: What version of the package were you running? - placeholder: "0.0.1" + placeholder: "{{cookiecutter._pkg_version}}" validations: required: true diff --git a/datajoint-workflow/{{cookiecutter.github_repo}}/.github/workflows/package-updates.yml b/datajoint-workflow/{{cookiecutter.github_repo}}/.github/workflows/package-updates.yml index 1e3dc2f..5cc521e 100644 --- a/datajoint-workflow/{{cookiecutter.github_repo}}/.github/workflows/package-updates.yml +++ b/datajoint-workflow/{{cookiecutter.github_repo}}/.github/workflows/package-updates.yml @@ -29,7 +29,7 @@ jobs: py_ver: "{{cookiecutter.python_version}}" release_branch: main - call_sciops_docker_images: + call_sciops_docker_image_debian: needs: call_semantic_versioning if: | always() && @@ -39,14 +39,13 @@ jobs: jhub_ver: "1.4.2" py_ver: "{{cookiecutter.python_version}}{% raw %}" dist: debian - env_base_hash: 44c5f19 - worker_base_hash: fcd8909 + codebook_base_hash: a7ee3ce workflow_version: ${{ needs.call_semantic_versioning.outputs.new_tag }} release_upload_url: ${{ needs.call_semantic_versioning.outputs.upload_url }} secrets: - RAW_DEPLOY_KEY: ${{ secrets.RAW_DEPLOY_KEY }} - REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} - REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + BOT_SSH_KEY: ${{secrets.BOT_SSH_KEY}} + REGISTRY_USERNAME: ${{secrets.REGISTRY_USERNAME}} + REGISTRY_PASSWORD: ${{secrets.REGISTRY_PASSWORD}} if_run_tests_failed: if: github.event.workflow_run.conclusion == 'failure' diff --git a/datajoint-workflow/{{cookiecutter.github_repo}}/README.md b/datajoint-workflow/{{cookiecutter.github_repo}}/README.md index 56d2873..1b75d5a 100644 --- a/datajoint-workflow/{{cookiecutter.github_repo}}/README.md +++ b/datajoint-workflow/{{cookiecutter.github_repo}}/README.md @@ -84,9 +84,48 @@ DataJoint LabBook displays data from your database. ... -### DataJoint Axon (Data Upload) +### DataJoint Axon (Data Upload/Download) + + +#### Axon GUI + +1. In the Axon app, on the lower left side, click the 'gear' icon (Config). Enter the following information in each field (See the `Misc` section below for what the generated `json` config file will look like once configured): + +- AWS Account ID: `123456789012` + +- DataJoint Account Client ID: `xxxxxxxxxxxxxxxxxxxxxx` + +- S3 Role: `{{cookiecutter.github_repo}}_researcher_prod` + +- S3 Bucket: `dj-sciops` + +2. At the top right of the Axon app, click the 'circle arrow' icon (Token). This will open a browser to be able to sign in with your DataJoint account and generate a temporary token. + +3. Select the 'S3 Bucket' tab on the left side of the Axon app. Enter the following information to view the current list of files uploaded, the press 'Load': + +- S3 Bucket Directory: `{{cookiecutter.github_repo}}/inbox/` + +4. To upload a folder to a subfolder within the 'S3 Bucket Directory', for example to upload the subject data folder 'SUBJ100' from your local machine, enter the following path on S3: `{{cookiecutter.github_repo}}/inbox/SUBJ100` + +#### Axon CLI + +... + +#### Misc + +```json +{ + "version": "1.2.0", + "aws": { "account_id": "123456789012" }, + "s3": + { + "role": "{{cookiecutter.github_repo}}_researcher_prod", + "bucket": "dj-sciops", + }, + "djauth": { "client_id": "xxxxxxxxxxxxxxxxxxxxxx" }, +} +``` -... ### DataJoint CodeBook (JupyterHub) diff --git a/datajoint-workflow/{{cookiecutter.github_repo}}/configs/axon.yaml b/datajoint-workflow/{{cookiecutter.github_repo}}/configs/axon.yaml deleted file mode 100644 index b0cf6da..0000000 --- a/datajoint-workflow/{{cookiecutter.github_repo}}/configs/axon.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# template for required information to use Axon. -version: "0.2.1" -aws: - account_id: "" -s3: - role: "" - bucket: "" -djauth: - client_id: "" diff --git a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/.datajoint_config.json b/datajoint-workflow/{{cookiecutter.github_repo}}/docker/.datajoint_config.json deleted file mode 100644 index 4b055aa..0000000 --- a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/.datajoint_config.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "connection.charset": "", - "connection.init_function": null, - "database.host": "localhost", - "database.password": null, - "database.port": 3306, - "database.reconnect": true, - "database.use_tls": false, - "database.user": null, - "display.limit": 12, - "display.show_tuple_count": true, - "display.width": 14, - "enable_python_native_blobs": true, - "fetch_format": "array", - "loglevel": "WARNING", - "safemode": true -} diff --git a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/.devcontainer/.env b/datajoint-workflow/{{cookiecutter.github_repo}}/docker/.devcontainer/.env new file mode 100644 index 0000000..3f6b019 --- /dev/null +++ b/datajoint-workflow/{{cookiecutter.github_repo}}/docker/.devcontainer/.env @@ -0,0 +1,6 @@ +COMPOSE_PROJECT_NAME=djsciops-devcontainer +DJ_PASS=simple +DJ_USER=root +REPO_OWNER={{cookiecutter.github_user}} +REPO_NAME={{cookiecutter.github_repo}} +WORKFLOW_VERSION={{cookiecutter._pkg_version}} diff --git a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/.devcontainer/devcontainer.json b/datajoint-workflow/{{cookiecutter.github_repo}}/docker/.devcontainer/devcontainer.json index eafa3c8..27149e5 100644 --- a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/.devcontainer/devcontainer.json +++ b/datajoint-workflow/{{cookiecutter.github_repo}}/docker/.devcontainer/devcontainer.json @@ -2,7 +2,7 @@ { "name": "{{cookiecutter.__project_name}} (devcontainer)", "dockerComposeFile": [ - "../docker-compose-devcontainer_env.yaml" + "./docker-compose.yml" ], // Add the IDs of extensions you want installed when the container is created. "extensions": [ @@ -15,7 +15,7 @@ "vivaxy.vscode-conventional-commits", "yzhang.markdown-all-in-one" ], - "initializeCommand": "test -f \"${localWorkspaceFolder}/.env\" || exit 7", + "initializeCommand": "test -f \"${localWorkspaceFolder}/.devcontainer/.env\" || exit 7", "service": "devcontainer_env", // Set *default* container specific settings.json values on container create. "settings": { @@ -35,8 +35,6 @@ "module", "docker" ], - "djcookiecutter.condaExec": "mamba", - "djcookiecutter.localPath": "${env:HOME}/Documents/datajoint/dj-cookiecutter", "files.exclude": { "**/.DS_Store": true, "**/.git": true, @@ -53,7 +51,7 @@ "build": true, "src/*.egg-info": true }, - "jupyter.alwaysScrollOnNewCell": true, + "interactiveWindow.alwaysScrollOnNewCell": true, "jupyter.generateSVGPlots": true, "jupyter.interactiveWindowMode": "single", "jupyter.notebookFileRoot": "${workspaceFolder}", @@ -65,7 +63,7 @@ "reportGeneralTypeIssues": "information" }, "python.analysis.typeCheckingMode": "basic", - "python.defaultInterpreterPath": "/usr/local/bin/python", + "python.defaultInterpreterPath": "/opt/local/conda/bin/python", "python.formatting.blackArgs": [ "--line-length=88" ], @@ -85,5 +83,5 @@ "python.testing.pytestEnabled": true }, "shutdownAction": "none", - "workspaceFolder": "/home/{{cookiecutter.github_user}}/{{cookiecutter.github_repo}}" + "workspaceFolder": "/home/ubuntu/{{cookiecutter.github_repo}}" } diff --git a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/docker-compose-devcontainer_env.yaml b/datajoint-workflow/{{cookiecutter.github_repo}}/docker/.devcontainer/docker-compose.yml similarity index 51% rename from datajoint-workflow/{{cookiecutter.github_repo}}/docker/docker-compose-devcontainer_env.yaml rename to datajoint-workflow/{{cookiecutter.github_repo}}/docker/.devcontainer/docker-compose.yml index f182bdc..e174769 100644 --- a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/docker-compose-devcontainer_env.yaml +++ b/datajoint-workflow/{{cookiecutter.github_repo}}/docker/.devcontainer/docker-compose.yml @@ -1,9 +1,9 @@ -# > docker-compose -f docker/docker-compose-devcontainer_env.yaml up -V --detach --force-recreate --remove-orphans --build -# > docker-compose -f docker/docker-compose-devcontainer_env.yaml down --volumes +# > docker-compose -f docker/.devcontainer/docker-compose.yml up -V --detach --force-recreate --remove-orphans --build +# > docker-compose -f docker/.devcontainer/docker-compose.yml down --volumes version: "2.4" services: - mysql_db: + mysql_sciops: environment: MYSQL_ROOT_PASSWORD: ${DJ_PASS:-simple} image: datajoint/mysql:5.7 @@ -14,21 +14,21 @@ services: - 6603:3306 restart: always volumes: - - sqldata:/var/lib/mysql + - sciops-sqldata:/var/lib/mysql devcontainer_env: build: - context: ../. - dockerfile: docker/devcontainer.Dockerfile + context: ../../. + dockerfile: docker/.devcontainer/build/Dockerfile args: - REPO_OWNER: - REPO_NAME: - image: registry.vathes.com/${REPO_OWNER}/devcontainer-${REPO_NAME}:py${PY_VER}-${DIST}-${WORKFLOW_VERSION} + REPO_NAME: ${REPO_NAME?} + WORKFLOW_VERSION: v${WORKFLOW_VERSION?} + image: ghcr.io/${REPO_OWNER?}/${REPO_NAME}:latest depends_on: - mysql_db: + mysql_sciops: condition: service_healthy environment: - DJ_HOST: mysql_db + DJ_HOST: mysql_sciops DJ_PASS: ${DJ_PASS:-simple} DJ_USER: ${DJ_USER:-root} init: true @@ -38,21 +38,20 @@ services: ports: - 18888:8888 volumes: - - ..:/home/${REPO_OWNER}/${REPO_NAME} - - vscode:/home/${REPO_OWNER}/.vscode-server/extensions - - vscode-insiders:/home/${REPO_OWNER}/.vscode-server-insiders/extensions - - /home/${REPO_OWNER}/${REPO_NAME}/.nox - - /home/${REPO_OWNER}/${REPO_NAME}/src/{{cookiecutter.__pkg_import_name}}.egg-info + - ../..:/home/ubuntu/${REPO_NAME} + - vscode:/home/ubuntu/.vscode-server/extensions + - /home/ubuntu/${REPO_NAME}/.nox + - /home/ubuntu/${REPO_NAME}/src/{{cookiecutter.__pkg_import_name}}.egg-info # MySQL GUI -------------- # http://localhost:8087/ # System: PostgreSQL - # Server: mysql_db + # Server: mysql_sciops # Username: root # Password: simple adminer: environment: - ADMINER_DEFAULT_SERVER: mysql_db + ADMINER_DEFAULT_SERVER: mysql_sciops ADMINER_DESIGN: nette image: adminer networks: @@ -62,7 +61,7 @@ services: published: 8087 restart: always depends_on: - mysql_db: + mysql_sciops: condition: service_started networks: @@ -70,9 +69,7 @@ networks: name: {{cookiecutter.__project_name}}-network volumes: - sqldata: - name: {{cookiecutter.__project_name}}-vol-sqldata + sciops-sqldata: + name: {{cookiecutter.__project_name}}-vol-sciops-sqldata vscode: name: {{cookiecutter.__project_name}}-vol-vscode - vscode-insiders: - name: {{cookiecutter.__project_name}}-vol-vscode-insiders diff --git a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/.gitignore b/datajoint-workflow/{{cookiecutter.github_repo}}/docker/.gitignore index a9640a5..6d7f971 100644 --- a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/.gitignore +++ b/datajoint-workflow/{{cookiecutter.github_repo}}/docker/.gitignore @@ -1,3 +1,6 @@ *.pem -.env -!.datajoint_config.json +*.env +!build.env +!run.example.env +!.devcontainer/.datajoint_config.json +!.devcontainer/.env diff --git a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/README.md b/datajoint-workflow/{{cookiecutter.github_repo}}/docker/README.md index 4bfe687..50220ba 100644 --- a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/README.md +++ b/datajoint-workflow/{{cookiecutter.github_repo}}/docker/README.md @@ -1,87 +1,65 @@ # Docker Containers -TODO: docker things -- test image builds for resuable workflows -- labhub multi conda envs -- actions checkout +See [djsciops-cicd](https://github.com/dj-sciops/djsciops-cicd) for more info. -Create a `.env` file and the change any variables as necessary, then source those variables for later use. +## Pushing an image to the datajoint container repository + +- check your remote ```bash -cat <<-EOF > ./docker/.env -COMPOSE_PROJECT_NAME={{cookiecutter.__project_name}} -JHUB_VER=1.4.2 -PY_VER={{cookiecutter.python_version}} -DIST=debian -DEPLOY_KEY={{cookiecutter.github_repo}}-deploy.pem -REPO_OWNER={{cookiecutter.github_user}} -REPO_NAME={{cookiecutter.github_repo}} -WORKFLOW_VERSION=v{{cookiecutter._pkg_version}} -HOST_UID=1000 -HOST_GID=1000 -DJ_HOST=host.docker.internal -DJ_USER=root -DJ_PASS=simple -EOF -set +a -source .env -set -a +git remote -v +# origin https://github.com/{{cookiecutter.github_user}}/{{cookiecutter.github_repo}} (fetch) +# origin https://github.com/{{cookiecutter.github_user}}/{{cookiecutter.github_repo}} (push) ``` -## CodeBook Environment - -### Docker Build and Run - -Change to the `docker` directory so that the docker build context is the current working directory. +- add upstream remote ```bash -cd docker -docker build \ - $(cat .env | while read li; do echo --build-arg ${li}; done | xargs) \ - --file codebook.Dockerfile \ - --tag registry.vathes.com/${REPO_OWNER}/codebook-${REPO_NAME}:jhub${JHUB_VER}-py${PY_VER}-${DIST}-${WORKFLOW_VERSION} \ - . +git remote add upstream https://github.com/dj-sciops/{{cookiecutter.github_repo}} +# origin https://github.com/{{cookiecutter.github_user}}/{{cookiecutter.github_repo}} (fetch) +# origin https://github.com/{{cookiecutter.github_user}}/{{cookiecutter.github_repo}} (push) +# upstream https://github.com/dj-sciops/{{cookiecutter.github_repo}} (fetch) +# upstream https://github.com/dj-sciops/{{cookiecutter.github_repo}} (push) ``` +- create a local branch from remote upstream/main and switch to it + ```bash -docker run -it \ - --platform linux/amd64 \ - --name {{cookiecutter.__project_name}} \ - --user root \ - registry.vathes.com/${REPO_OWNER}/codebook-${REPO_NAME}:jhub${JHUB_VER}-py${PY_VER}-${DIST}-${WORKFLOW_VERSION} \ - bash +git fetch upstream +git switch -c upstream-main upstream/main ``` -### Docker Compose - -Will automatically load environment variables from `.env` file. +- push a tag ```bash -cd docker -docker-compose -f docker-compose-codebook_env.yaml up --detach --force-recreate --remove-orphans --build +git tag 0.1.1 # recreate local tag +git push upstream 0.1.1 # recreate remote tag and trigger CICD ``` -## Standard Worker Environment - -TODO: add description +- if something went wrong, delete a tag +```bash +git push upstream --delete 0.1.1 # delete remote tag +git tag -d 0.1.1 # delete local tag +``` -### Docker Compose - -Will automatically load environment variables from `.env` file. +- switch back to your local main branch ```bash -cd docker -docker-compose -f docker-compose-standard_worker.yaml up --detach --force-recreate --remove-orphans --build +git switch main ``` +## Standard Worker Environment + +TODO: add description + ## Devcontainer Environment > **Note**: Make sure to first set the environment variables file `.env` as outlined at the top of this document, otherwise building the docker image will fail. ### VSCode -You can either open to the docker folder directly from vscode and it'll prompt you to open the devcontainer, or use the supplied script below. +You can either open to the docker folder directly from vscode and it'll prompt you to open the devcontainer, or use the supplied script below. ```bash chmod +x docker/.devcontainer/vscode-open-dev-container @@ -93,6 +71,36 @@ docker/.devcontainer/vscode-open-dev-container Will automatically load environment variables from `.env` file. ```bash -cd docker -docker-compose -f docker-compose-devcontainer_env.yaml up --detach --force-recreate --remove-orphans --build +cd docker/.devcontainer +docker-compose up --detach --force-recreate --remove-orphans --build +``` + +### Docker Build and Run + +Example building one of the stages from the Dockerfile + +**Stage**: `micromamba_debian` + +```bash +cd docker/.devcontainer/build +PLTARCH=$(uname -m) # target architecture for platform +MSTARG=micromamba_debian # multi-stage build target name +IMGTAG=devcontainer:v0.0.1 # image tag +DCNAME=sciops-devcontainer # container name +DCUSER=ubuntu # container user +``` + +```bash +# build +docker build --platform=linux/${PLTARCH?} --target=$MSTARG --tag=$IMGTAG . + +# start container +docker run --rm -itdu "$DCUSER:docker" --name $DCNAME $IMGTAG bash + +# execute a command to running container as root +DCID=$(docker ps -aqf "name=$DCNAME") +docker exec -u "root:docker" ${DCID?} chown $DCUSER:docker /usr/local/conda-meta/history + +# stop container +docker stop $DCNAME ``` diff --git a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/apt_requirements.txt b/datajoint-workflow/{{cookiecutter.github_repo}}/docker/apt_requirements.txt deleted file mode 100644 index afe6096..0000000 --- a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/apt_requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -build-essential -bzip2 -curl -git -net-tools -openssh-client -procps -vim -wget diff --git a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/codebook.Dockerfile b/datajoint-workflow/{{cookiecutter.github_repo}}/docker/codebook.Dockerfile deleted file mode 100644 index 339bb7d..0000000 --- a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/codebook.Dockerfile +++ /dev/null @@ -1,48 +0,0 @@ -ARG JHUB_VER=1.4.2 -ARG PY_VER={{cookiecutter.python_version}} -ARG DIST=debian - -FROM datajoint/djlabhub:${JHUB_VER}-py${PY_VER}-${DIST} - -ARG DEPLOY_KEY={{cookiecutter.github_repo}}-deploy.pem -ARG REPO_OWNER={{cookiecutter.github_user}} -ARG REPO_NAME={{cookiecutter.github_repo}} -ARG WORKFLOW_VERSION=main - -USER root:anaconda - -COPY $DEPLOY_KEY /root/.ssh/sciops_deploy.ssh -COPY --chown=anaconda:anaconda ["pip_requirements.txt*", "apt_requirements.txt*", "conda_requirements.txt*", "nofile.txt", "/tmp/"] - -RUN apt-get -y update &&\ - [ -f /tmp/apt_requirements.txt ] && xargs >/root/.ssh/known_hosts &&\ - mkdir -p /tmp/${REPO_NAME} &&\ - GIT_SSH_COMMAND="ssh -vvv -i /root/.ssh/sciops_deploy.ssh" git clone --branch ${WORKFLOW_VERSION} --single-branch git@github.com:${REPO_OWNER}/${REPO_NAME}.git /tmp/${REPO_NAME} &&\ - chown -R anaconda:anaconda /tmp/${REPO_NAME} &&\ - cp -f /tmp/${REPO_NAME}/docker/.datajoint_config.json /usr/local/bin/.datajoint_config.json &&\ - cp -fR /tmp/${REPO_NAME}/notebooks /home/ &&\ - cp -f /tmp/${REPO_NAME}/notebooks/README* /home/ &&\ - cp -fR /tmp/${REPO_NAME}/scripts /home/common/ &&\ - chown -R :anaconda /home &&\ - chmod -R 2770 /home &&\ - chmod 664 /tmp/djlab_config.yaml &&\ - conda install -y 'mamba' &&\ - rm -rf /root/.ssh/sciops_deploy.ssh /tmp/apt_requirements.txt /var/lib/apt/lists/* /home/notebooks/README* - -# Note: permission issues with djlab module, .ipython folder, ipykernel with custom env. - -USER anaconda:anaconda -RUN mamba env create -qv -f /tmp/${REPO_NAME}/environment.yml &&\ - conda run --no-capture-output -n {{cookiecutter.__pkg_import_name}} pip install /tmp/${REPO_NAME} - -USER root:anaconda -RUN conda run --no-capture-output -n {{cookiecutter.__pkg_import_name}} ipython kernel install --name={{cookiecutter.__pkg_import_name}} &&\ - rm -fR /tmp/${REPO_NAME} &&\ - mamba clean -ya - -USER anaconda:anaconda -WORKDIR /home/notebooks diff --git a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/devcontainer.Dockerfile b/datajoint-workflow/{{cookiecutter.github_repo}}/docker/devcontainer.Dockerfile deleted file mode 100644 index af2e136..0000000 --- a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/devcontainer.Dockerfile +++ /dev/null @@ -1,124 +0,0 @@ -# syntax=docker/dockerfile:1 - -ARG PROJECT_NAME={{cookiecutter.__project_name}} -ARG IMPORT_NAME={{cookiecutter.__pkg_import_name}} -ARG REPO_OWNER={{cookiecutter.github_user}} -ARG REPO_NAME={{cookiecutter.github_repo}} -ARG HOST_UID=1000 -ARG HOST_GID=1000 -ARG USER_SUDO=true -ARG IMAGE_DATE=2021-11-11T11:11:11Z -ARG WORKFLOW_VERSION=v0.0.1 -ARG TIMEZONE="America/Chicago" - -# Stage 1 ============================================================================== -# docker build -f devcontainer.Dockerfile --tag wf_img_1 --target build_pyenv .. -# docker run -itd --user root --name wf_build_pyenv wf_img_1 bash -FROM ghcr.io/iamamutt/conda_base:latest as build_pyenv - -ARG REPO_NAME -ARG PROJECT_NAME -ARG HOST_GID -WORKDIR /usr/local/src/${REPO_NAME} -COPY ./ ./ -SHELL [ "/bin/bash", "-ec" ] - -RUN <<-EOF - cp -f docker/apt_requirements.txt ../apt_requirements.txt - cp -f docker/.datajoint_config.json ../.datajoint_config.json - init-apt-deps ../apt_requirements.txt - init-conda-env environment.yml - source ~/.bashrc - chmod -R 2775 /usr/local - CONDA_ENV_USER=base conda-run mamba install -yq conda-pack - CONDA_ENV_USER=base conda-run conda pack -n ${CONDA_ENV_USER} -o condaenv.tar.gz - tar -xzf condaenv.tar.gz -C /usr/local - source /usr/local/bin/activate - conda-unpack - rm -rf /usr/local/src/${REPO_NAME} - chmod -R g+wX /usr/local -EOF - -ENTRYPOINT [ "/bin/sh", "-c" ] -CMD [ "tail", "-f", "/dev/null" ] - -# Stage 2 ============================================================================== -# docker build -f devcontainer.Dockerfile --tag wf_img_2 --target install_pyenv .. -# docker run -itd --user root --name wf_install_pyenv wf_img_2 bash -FROM debian:11-slim as install_pyenv - -ARG REPO_OWNER -ARG REPO_NAME -ARG HOST_UID -ARG HOST_GID -ARG USER_SUDO -ARG TIMEZONE - -ENV NEW_USER_NAME=${REPO_OWNER} -ENV NEW_USER_GROUP=${REPO_OWNER} -ENV NEW_USER_UID=${HOST_UID} -ENV NEW_USER_GID=${HOST_GID} -ENV NEW_USER_SUDO=${USER_SUDO} -ENV WORKFLOW_DIR=/home/${REPO_OWNER}/${REPO_NAME} -ENV TZ=${TIMEZONE} -ENV LANG=en_US.UTF-8 -ENV LC_ALL=en_US.UTF-8 - -SHELL [ "/bin/bash", "-ec" ] -COPY --from=build_pyenv --chown=${NEW_USER_UID}:${NEW_USER_GID} /usr/local/ /usr/local/ - -RUN <<-EOF - mkdir -p -m 2775 /usr/share/man/man1 - echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections - apt-get update - apt-get -qq install --no-install-recommends -y apt-utils locales ca-certificates - localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 - ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime - echo ${TZ} >/etc/timezone - [[ ${NEW_USER_SUDO} = true ]] && echo "sudo" >>/usr/local/src/apt_requirements.txt - init-apt-deps /usr/local/src/apt_requirements.txt - init-new-user - mkdir -p "${WORKFLOW_DIR}" - chown ${NEW_USER_NAME}:${NEW_USER_GROUP} "${WORKFLOW_DIR}" /usr/local -EOF - -WORKDIR "${WORKFLOW_DIR}" -COPY --chown=${NEW_USER_NAME}:${NEW_USER_GROUP} ./ ./ -USER ${NEW_USER_NAME}:${NEW_USER_GROUP} - -RUN <<-EOF - rm -rf .nox .git .mypy_cache .pytest_cache site build - cp -f /usr/local/src/.datajoint_config.json ../.datajoint_config.json - mkdir -p .nox src/{{cookiecutter.__pkg_import_name}}.egg-info \ - ../.vscode-server/extensions ../.vscode-server-insiders/extensions - chmod -R 2775 ../.vscode-server* - source activate - conda init -q bash - pip install --no-cache-dir -e '.[dev,doc,test,sciops]' -EOF - -ENTRYPOINT [ "/bin/sh", "-c" ] -CMD [ "tail", "-f", "/dev/null" ] - -# Stage 3 ============================================================================== -# docker build -f devcontainer.Dockerfile --tag wf_img_3 --target devcontainer .. -# docker run -itd --user root --name wf_devcontainer wf_img_3 bash -FROM scratch as devcontainer -COPY --from=install_pyenv / / -ARG IMAGE_DATE -ARG WORKFLOW_VERSION -ARG REPO_OWNER -ARG REPO_NAME -ARG TIMEZONE -ENV TZ=${TIMEZONE} -ENV LANG=en_US.UTF-8 -ENV LC_ALL=en_US.UTF-8 -LABEL org.opencontainers.image.authors "Joseph M. Burling" -LABEL org.opencontainers.image.title "{{cookiecutter.github_repo}}" -LABEL org.opencontainers.image.description "A development container with a debian-based python environment" -LABEL org.opencontainers.image.version "$WORKFLOW_VERSION" -LABEL org.opencontainers.image.created "$IMAGE_DATE" -USER ${REPO_OWNER}:${REPO_OWNER} -WORKDIR /home/${REPO_OWNER}/${REPO_NAME} -ENTRYPOINT [ "/bin/bash", "-lc" ] -CMD [ "tail", "-f", "/dev/null" ] diff --git a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/docker-compose-codebook_env.yaml b/datajoint-workflow/{{cookiecutter.github_repo}}/docker/docker-compose-codebook_env.yaml deleted file mode 100644 index 475c6b0..0000000 --- a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/docker-compose-codebook_env.yaml +++ /dev/null @@ -1,33 +0,0 @@ -version: "2.4" -services: - codebook_env: - build: - context: . - dockerfile: codebook.Dockerfile - args: - JHUB_VER: - PY_VER: - DIST: - DEPLOY_KEY: - REPO_OWNER: - REPO_NAME: - WORKFLOW_VERSION: - image: registry.vathes.com/${REPO_OWNER}/codebook-${REPO_NAME}:jhub${JHUB_VER}-py${PY_VER}-${DIST}-${WORKFLOW_VERSION} - environment: - DISPLAY: # Necessary to allow GUI to route to Docker host - NEW_USER: ${REPO_OWNER:-{{cookiecutter.github_user}}} # Rename user and adjust HOME directory - DJ_HOST: ${DJ_HOST:-host.docker.internal} # Specify DataJoint database host - DJ_USER: ${DJ_USER:-root} # Specify DataJoint database user - DJ_PASS: ${DJ_PASS:-simple} # Specify DataJoint database password - Djlab_JupyterServer_Password: datajoint # Jupyter login password - ports: - - 8888:8888 - volumes: - - /tmp/.X11-unix:/tmp/.X11-unix:rw # Necessary to allow GUI to route to Docker host - user: ${HOST_UID:-1000}:anaconda # Necessary to allow GUI to route to Docker host - command: jupyter lab - # user: root:anaconda - # init: true - # tty: true - # entrypoint: ["/bin/sh", "-c"] - # command: tail -f /dev/null diff --git a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/docker-compose-standard_workflow.yaml b/datajoint-workflow/{{cookiecutter.github_repo}}/docker/docker-compose-standard_workflow.yaml deleted file mode 100644 index b9ad993..0000000 --- a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/docker-compose-standard_workflow.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# docker-compose up --build --detach -version: "2.4" -services: - standard_worker: - build: # only necessary if rebuilding image - context: . - dockerfile: standard_worker.Dockerfile - args: - - DEPLOY_KEY - image: registry.vathes.com/${REPO_OWNER}/standard-${REPO_NAME}:py${PY_VER}-${DIST}-${WORKFLOW_VERSION} - environment: - NEW_USER: ${REPO_OWNER:-{{cookiecutter.github_user}}} # Rename user and adjust HOME directory - DJ_HOST: ${DJ_HOST:-host.docker.internal} # Specify DataJoint database host - DJ_USER: ${DJ_USER:-root} # Specify DataJoint database user - DJ_PASS: ${DJ_PASS:-simple} # Specify DataJoint database password - DATABASE_PREFIX: {{cookiecutter.__pkg_import_name}} # Speicify DataJoint database prefix - volumes: - - ${ROOT_DATA_DIR:-../data/raw}:/home/${REPO_OWNER:-{{cookiecutter.github_user}}}/workflow_root_data - - ${PROCESSED_DATA_DIR:-../data/preprocessed}:/home/${REPO_OWNER:-{{cookiecutter.github_user}}}/workflow_processed_data - - ./apt_requirements.txt:/tmp/apt_requirements.txt - scale: ${WORKER_COUNT:-1} - command: - - /bin/bash - - -c - - | - {{cookiecutter.__pkg_import_name}}_entrypoint --help & - tail -f /dev/null diff --git a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/nofile.txt b/datajoint-workflow/{{cookiecutter.github_repo}}/docker/nofile.txt deleted file mode 100644 index bca9069..0000000 --- a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/nofile.txt +++ /dev/null @@ -1 +0,0 @@ -# This empty file is intentionally left blank and must exist in case *_requirements.txt files are not present in the context directory. diff --git a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/standard_worker.Dockerfile b/datajoint-workflow/{{cookiecutter.github_repo}}/docker/standard_worker.Dockerfile deleted file mode 100644 index e534c64..0000000 --- a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/standard_worker.Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM datajoint/djbase:latest - -USER root -RUN apt update &&\ - apt-get install -y ssh git - -USER anaconda:anaconda -ARG DEPLOY_KEY -COPY --chown=anaconda $DEPLOY_KEY $HOME/.ssh/sciops_deploy.ssh -WORKDIR $HOME -RUN ssh-keyscan github.com >> ~/.ssh/known_hosts &&\ - GIT_SSH_COMMAND="ssh -i $HOME/.ssh/sciops_deploy.ssh"\ - git clone git@github.com:{{cookiecutter.github_user}}/{{cookiecutter.github_repo}}.git - -RUN cd "$HOME/{{cookiecutter.github_repo}}" &&\ - pip install '.[sciops]' diff --git a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/standard_worker/build.env b/datajoint-workflow/{{cookiecutter.github_repo}}/docker/standard_worker/build.env new file mode 100644 index 0000000..7e6b040 --- /dev/null +++ b/datajoint-workflow/{{cookiecutter.github_repo}}/docker/standard_worker/build.env @@ -0,0 +1,6 @@ +PY_VER={{cookiecutter.python_version}} +WORKER_BASE_HASH=8eb1715 +WORKFLOW_VERSION={{cookiecutter._pkg_version}} +DEPLOY_KEY={{cookiecutter.github_repo}}-deploy.pem +REPO_OWNER={{cookiecutter.github_user}} +REPO_NAME={{cookiecutter.github_repo}} diff --git a/datajoint-workflow/{{cookiecutter.github_repo}}/docker/standard_worker/run.example.env b/datajoint-workflow/{{cookiecutter.github_repo}}/docker/standard_worker/run.example.env new file mode 100644 index 0000000..e69de29