improve the building system

.github/workflows/ci_code.yml

@@ -76,6 +76,9 @@ jobs:
       - name: Integration Testing
         if: ${{ startsWith(matrix.python-version, '3.11') }} # Dask requires local python and Dockerfile to be in sync
         run: |
+          # Update hostnames
+          echo  127.0.0.1 mongodb | sudo tee -a /etc/hosts
+
           # Run testing environment on the background, but get the logs
           make testenv_init
 

Makefile

@@ -77,19 +77,20 @@ hr-docs: ## Generate Docusaurus documentation and blog posts
 ##@ CI Testing Environment
 
 testenv_init: ## Initialize a local Testing environment
+	@echo "===> Ensure hostnames"
+	@deploy/testenv/validate_hostnames.sh
+
 	@echo "===> Build superduperdb/sandbox"
-	docker build . -f ./images/superduperdb/Dockerfile -t superduperdb/sandbox --progress=plain \
+	docker build . -f deploy/images/superduperdb/Dockerfile -t superduperdb/sandbox --progress=plain \
 		--build-arg BUILD_ENV="sandbox" \
 		--build-arg SUPERDUPERDB_EXTRAS="dev"
 
-	@echo "===> Updating host files"
-	./test/material/testenv/set_hosts.sh
-
 	@echo "===> Run Docker-Compose using superduperdb/sandbox"
-	docker compose -f test/material/testenv/docker-compose.yaml up --remove-orphans &
+	docker compose -f deploy/testenv/docker-compose.yaml up --remove-orphans &
 
 testenv_shutdown: ## Terminate the local Testing environment
-	docker compose -f test/material/testenv/docker-compose.yaml down
+	@echo "===> Shutting down the local Testing environment"
+	docker compose -f deploy/testenv/docker-compose.yaml down
 
 ##@ CI Testing Functions
 
@@ -98,7 +99,7 @@ unit-testing: ## Execute unit testing
 
 integration-testing: ## Execute integration testing
 	# Block waiting for the testenv to become ready.
-	cd ./test/material/testenv/; ./wait_ready.sh
+	@cd deploy/testenv/; ./wait_ready.sh
 
 	# Run the test
 	pytest $(PYTEST_ARGUMENTS) ./test/integration
@@ -175,8 +176,8 @@ run_sandbox-pr: ## Run a pull request in the sandbox (argument: PR_NUMBER=555)
 # superduperdb/superduperdb is a minimal image contains only what is needed for the framework.
 build_superduperdb: ## Build a minimal Docker image for general use
 	echo "===> build superduperdb/superduperdb:$(RELEASE_VERSION:v%=%)"
-	docker build . -f ./images/superduperdb/Dockerfile -t superduperdb/superduperdb:$(RELEASE_VERSION:v%=%) --progress=plain --no-cache \
-	--build-arg BUILD_ENV="pypi"
+	docker build . -f ./deploy/images/superduperdb/Dockerfile -t superduperdb/superduperdb:$(RELEASE_VERSION:v%=%) --progress=plain --no-cache \
+	--build-arg BUILD_ENV="release"
 
 
 push_superduperdb: ## Push the superduperdb/superduperdb:latest image
@@ -193,8 +194,8 @@ push_superduperdb: ## Push the superduperdb/superduperdb:latest image
 # superduperdb/demo is a bloated image that contains everything we need to run the online demo.
 build_demo: ## Build a feature-rich Docker image for demonstrations
 	echo "===> build superduperdb/demo:$(RELEASE_VERSION:v%=%)"
-	docker build . -f ./images/superduperdb/Dockerfile -t superduperdb/demo:$(RELEASE_VERSION:v%=%) --progress=plain --no-cache \
-	--build-arg BUILD_ENV="pypi" \
+	docker build . -f ./deploy/images/superduperdb/Dockerfile -t superduperdb/demo:$(RELEASE_VERSION:v%=%) --progress=plain --no-cache \
+	--build-arg BUILD_ENV="release" \
 	--build-arg SUPERDUPERDB_EXTRAS="demo"
 
 push_demo: ## Push the superduperdb/demo:latest image

images/superduperdb/Dockerfile → deploy/images/superduperdb/Dockerfile

@@ -1,19 +1,16 @@
 # syntax = docker/dockerfile:experimental
 
+# ---------------
+# Global Parameters
+# ---------------
 # BUILD_ENV choses whether to use local source (sandbox) or released wheel (pypi).
-ARG BUILD_ENV=pypi
+ARG BUILD_ENV=release
 
 # ---------------
 # Configure Basic Template
 # ---------------
 FROM jupyterhub/k8s-singleuser-sample:3.1.0 as base
 
-ARG SUPERDUPERDB_EXTRAS='" "'
-
-ENV NB_USER=superduper \
-    NB_UID=1000 \
-    HOME=/home/superduper
-
 # Temporarily switch to the root for privileged operations.
 # ---------------
 USER root
@@ -36,6 +33,10 @@ RUN apt-get update \
 
 # Replace the default Jupyter user with a SuperDuper user
 # ---------------
+ENV NB_USER=superduper \
+    NB_UID=1000 \
+    HOME=/home/superduper
+
 RUN deluser jovyan && rm -rf /home/jovyan \
     && adduser \
         --disabled-password \
@@ -56,7 +57,7 @@ ENV PATH="${HOME}/.local/bin:$PATH"
 # Install common dependencies
 # ---------------
 RUN pip install --upgrade setuptools pip
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip pip install --upgrade --user \
+RUN --mount=type=cache,uid=1000,target=/home/superduper/.cache/pip pip install --upgrade --user \
     # JupyterLab extensions \
     theme-darcula \
     ipywidgets \
@@ -67,32 +68,34 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip pip install --upgrade -
 
 # Install Jupyterlab extensions
 # ---------------
-COPY --chown=superduper ./images/superduperdb/labextensions/@superduperdb ${HOME}/.local/share/jupyter/labextensions/@superduperdb
-COPY --chown=superduper ./images/superduperdb/apputils-extension/themes.jupyterlab-settings ${HOME}/.jupyter/lab/user-settings/@jupyterlab/apputils-extension/themes.jupyterlab-settings
-
+COPY --chown=superduper ./deploy/images/superduperdb/labextensions/@superduperdb ${HOME}/.local/share/jupyter/labextensions/@superduperdb
+COPY --chown=superduper ./deploy/images/superduperdb/apputils-extension/themes.jupyterlab-settings ${HOME}/.jupyter/lab/user-settings/@jupyterlab/apputils-extension/themes.jupyterlab-settings
 
 # ---------------
-# Configure Sandbox Build
+# Build Sandbox
 # ---------------
 FROM base AS build_sandbox
+
+ONBUILD ARG SUPERDUPERDB_EXTRAS="' '"
 ONBUILD COPY --chown=superduper ./ ${HOME}/superduperdb
 ONBUILD WORKDIR ${HOME}/superduperdb
-ONBUILD RUN --mount=type=cache,mode=0755,target=/root/.cache/pip pip install --upgrade --user \
-    .[${SUPERDUPERDB_EXTRAS}]    \
-    # Purge pip cache
-    && pip cache purge
-
+# Do our best for caching.
+ONBUILD RUN --mount=type=cache,uid=1000,target=/home/superduper/.cache/pip pip install --upgrade --user \
+    .[${SUPERDUPERDB_EXTRAS}]
 
 # ---------------
-# Configure pypi Build
+# Build Release
 # ---------------
-FROM base AS build_pypi
+FROM base AS build_release
+
+ONBUILD ARG SUPERDUPERDB_EXTRAS="' '"
 ONBUILD COPY --chown=superduper ${PWD}/examples ./examples
-ONBUILD RUN --mount=type=cache,mode=0755,target=/root/.cache/pip pip install --upgrade --user \
+# Drop cache to reduce image size.
+ONBUILD RUN pip install --upgrade --user \
     superduperdb[${SUPERDUPERDB_EXTRAS}] \
     # Purge pip cache
     && pip cache purge
-
+ONBUILD WORKDIR ${HOME}/examples
 
 # ---------------
 # Select Build

test/material/testenv/docker-compose.yaml → deploy/testenv/docker-compose.yaml

@@ -7,8 +7,8 @@ services:
   mongodb:
     image: mongo:6
     hostname: mongodb
-    environment:
-      - MONGODB_ADVERTISED_HOSTNAME=localhost
+#    environment:
+#      - MONGODB_ADVERTISED_HOSTNAME=localhost
     ports:
       - "27017:27017"
     command : ["/bin/sh", "-c", "mongod --replSet rs0 --port 27017 --bind_ip 0.0.0.0 --dbpath /data/db/"]
@@ -34,11 +34,15 @@ services:
     hostname: vector-search
     ports:
       - "8000:8000"
+    volumes:
+      - ../../:/home/superduper/superduperdb # mount the project path
+    environment:
+      SUPERDUPERDB_DATA_BACKEND: 'mongodb://superduper:superduper@mongodb:27017/test_db'
     command:
       - /bin/sh
       - -c
       - |
-        SUPERDUPERDB_DATA_BACKEND='mongodb://superduper:superduper@mongodb:27017/test_db' python -m superduperdb vector-search
+        python -m superduperdb vector-search
 
     healthcheck:
       test: curl http://localhost:8000/health || exit 1
@@ -64,11 +68,15 @@ services:
     hostname: cdc
     ports:
       - "8001:8001"
+    volumes:
+      - ../../:/home/superduper/superduperdb # mount the project path
+    environment:
+      SUPERDUPERDB_DATA_BACKEND: 'mongodb://superduper:superduper@mongodb:27017/test_db'
     command:
       - /bin/sh
       - -c
       - |
-        SUPERDUPERDB_DATA_BACKEND='mongodb://superduper:superduper@mongodb:27017/test_db' python -m superduperdb cdc
+        python -m superduperdb cdc
 
     healthcheck:
       test: curl http://localhost:8001/health || exit 1
@@ -95,21 +103,29 @@ services:
     ports:
       - "8786:8786" # Peer communication
       - "8787:8787" # HTTP Dashboard
+    volumes:
+      - ../../:/home/superduper/superduperdb # mount the project path
+    environment:
+      SUPERDUPERDB_DATA_BACKEND: 'mongodb://superduper:superduper@mongodb:27017/test_db'
     command:
       - /bin/sh
       - -c
       - |
-        SUPERDUPERDB_DATA_BACKEND='mongodb://superduper:superduper@mongodb:27017/test_db' dask scheduler
+        dask scheduler
 
   dask-worker:
     depends_on:
       -   dask-scheduler
     image: superduperdb/sandbox
+    volumes:
+      - ../../:/home/superduper/superduperdb # mount the project path
+    environment:
+      SUPERDUPERDB_DATA_BACKEND: 'mongodb://superduper:superduper@mongodb:27017/test_db'
     command:
       - /bin/sh
       - -c
       - |
-        SUPERDUPERDB_DATA_BACKEND='mongodb://superduper:superduper@mongodb:27017/test_db' dask worker "tcp://scheduler:8786"
+        dask worker "tcp://scheduler:8786"
     deploy:
       replicas: 1
 
@@ -122,7 +138,7 @@ services:
 #      - dask-scheduler
 #    build: # Build the latest SuperDuperDB code
 #      context: ../../../. # Build from project's root
-#      dockerfile: ./images/superduperdb/Dockerfile
+#      dockerfile: ./deploy/images/superduperdb/Dockerfile
 #      args:
 #        - BUILD_ENV=sandbox # Use local code
 #        - SUPERDUPERDB_EXTRAS=demo # Install dependencies

deploy/testenv/validate_hostnames.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+
+# Ensure mongodb entry
+if grep -q '127.0.0.1.*mongodb' /etc/hosts
+then
+   echo "mongodb found";
+else
+  echo "****************************"
+  echo -e "You need to update /etc/hosts:\n"
+  echo "echo 127.0.0.1 mongodb | sudo tee -a /etc/hosts"
+  echo "****************************"
+  exit 255
+fi