WIP refactor container and queuing system

.github/ISSUE_TEMPLATE/bug_report.md

@@ -16,10 +16,10 @@ Note: Please search to see if an issue already exists for the bug you encountere
 A clear and concise description of what the bug is.
 
 **Task List**
-[ ] I have read thoroughly the README setup and installation instructions.
-[ ] If my setup is different, I have included my docker-compose file.
-[ ] I have included my redacted `.env` file.
-[ ] I have included information on my machine, and environment.
+- [ ] I have read thoroughly the README setup and installation instructions.
+- [ ] If my setup is different, I have included my docker-compose file.
+- [ ] I have included my redacted `.env` file.
+- [ ] I have included information on my machine, and environment.
 
 **To Reproduce**
 Steps to reproduce the behavior:

.github/workflows/build_push_docker_latest.yml

@@ -4,17 +4,16 @@ on:
   workflow_dispatch:
   push:
     branches: [main]
-  pull_request:
-    branches: [main]
 
 jobs:
-  build_and_push_server_latest:
+  # This image include both the server and microservices - the two containers can be slitted into separated
+  # service with its coressponding entry file.
+  build_and_push_server_monorepo_latest:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
         with:
-          # ref: "main" # branch
           fetch-depth: 0
 
       - name: Set up QEMU
@@ -27,23 +26,22 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build and push Immich
+      - name: Build and push Immich Mono Repo
         uses: docker/[email protected]
         with:
           context: ./server
           file: ./server/Dockerfile
           platforms: linux/arm/v7,linux/amd64,linux/arm64
-          push: ${{ github.event_name != 'pull_request' }}
+          push: true
           tags: |
             altran1502/immich-server:latest
 
-  build_and_push_microservice_latest:
+  build_and_push_machine_learning_latest:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
         with:
-          # ref: "main" # branch
           fetch-depth: 0
 
       - name: Set up QEMU
@@ -56,23 +54,22 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build and Push Microservices
+      - name: Build and Push Machine Learning
         uses: docker/[email protected]
         with:
-          context: ./microservices
-          file: ./microservices/Dockerfile
+          context: ./machine-learning
+          file: ./machine-learning/Dockerfile
           platforms: linux/arm/v7,linux/amd64
-          push: ${{ github.event_name != 'pull_request' }}
+          push: true
           tags: |
-            altran1502/immich-microservices:latest
+            altran1502/immich-machine-learning:latest
 
   build_and_push_web_latest:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
         with:
-          # ref: "main" # branch
           fetch-depth: 0
       - name: Set up QEMU
         uses: docker/[email protected]
@@ -91,6 +88,6 @@ jobs:
           file: ./web/Dockerfile
           platforms: linux/arm/v7,linux/amd64,linux/arm64
           target: prod
-          push: ${{ github.event_name != 'pull_request' }}
+          push: true
           tags: |
             altran1502/immich-web:latest

.github/workflows/build_push_docker_staging.yml

@@ -0,0 +1,95 @@
+name: Build and Push Docker Image - Staging
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  # This image include both the server and microservices - the two containers can be slitted into separated
+  # service with its coressponding entry file.
+  build_and_push_server_monorepo_staging:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set up QEMU
+        uses: docker/[email protected]
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/[email protected]
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push Immich Mono Repo
+        uses: docker/[email protected]
+        with:
+          context: ./server
+          file: ./server/Dockerfile
+          platforms: linux/arm/v7,linux/amd64,linux/arm64
+          push: ${{ github.event_name == 'pull_request' }}
+          tags: |
+            altran1502/immich-server:staging
+
+  build_and_push_machine_learning_staging:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set up QEMU
+        uses: docker/[email protected]
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/[email protected]
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and Push Machine Learning
+        uses: docker/[email protected]
+        with:
+          context: ./machine-learning
+          file: ./machine-learning/Dockerfile
+          platforms: linux/arm/v7,linux/amd64
+          push: ${{ github.event_name == 'pull_request' }}
+          tags: |
+            altran1502/immich-machine-learning:staging
+
+  build_and_push_web_staging:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up QEMU
+        uses: docker/[email protected]
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/[email protected]
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and Push Web
+        uses: docker/[email protected]
+        with:
+          context: ./web
+          file: ./web/Dockerfile
+          platforms: linux/arm/v7,linux/amd64,linux/arm64
+          target: prod
+          push: ${{ github.event_name == 'pull_request' }}
+          tags: |
+            altran1502/immich-web:staging

docker/docker-compose.dev.yml

@@ -6,7 +6,7 @@ services:
     build:
       context: ../server
       dockerfile: Dockerfile
-    command: npm run start:dev
+    command: npm run start:dev immich
     expose:
       - "3000"
     volumes:
@@ -23,16 +23,35 @@ services:
     networks:
       - immich-network
 
-  immich-microservices:
-    image: immich-microservices-dev:1.9.0
+  immich-machine-learning:
+    image: immich-machine-learning-dev:1.9.0
     build:
-      context: ../microservices
+      context: ../machine-learning
       dockerfile: Dockerfile
     command: npm run start:dev
     expose:
       - "3001"
     volumes:
-      - ../microservices:/usr/src/app
+      - ../machine-learning:/usr/src/app
+      - ${UPLOAD_LOCATION}:/usr/src/app/upload
+      - /usr/src/app/node_modules
+    env_file:
+      - .env
+    environment:
+      - NODE_ENV=development
+    depends_on:
+      - database
+    networks:
+      - immich-network
+
+  immich-microservices:
+    image: immich-microservices:1.9.0
+    build:
+      context: ../server
+      dockerfile: Dockerfile
+    command: npm run start:dev microservices
+    volumes:
+      - ../server:/usr/src/app
       - ${UPLOAD_LOCATION}:/usr/src/app/upload
       - /usr/src/app/node_modules
     env_file:
@@ -44,6 +63,7 @@ services:
     networks:
       - immich-network
 
+
   immich-web:
     image: immich-web-dev:1.9.0
     build:

docker/docker-compose.staging.yml

@@ -2,11 +2,11 @@ version: "3.8"
 
 services:
   immich-server:
-    image: immich-server-staging:latest
+    image: immich-server:staging
     build:
       context: ../server
       dockerfile: Dockerfile
-    entrypoint: ["/bin/sh", "./entrypoint.sh"]
+    entrypoint: ["/bin/sh", "./start-server.sh"]
     expose:
       - "3000"
     volumes:
@@ -23,11 +23,11 @@ services:
     restart: always
 
   immich-microservices:
-    image: immich-microservices-staging:latest
+    image: immich-microservices:staging
     build:
-      context: ../microservices
+      context: ../server
       dockerfile: Dockerfile
-    entrypoint: ["/bin/sh", "./entrypoint.sh"]
+    entrypoint: ["/bin/sh", "./start-microservices.sh"]
     expose:
       - "3001"
     volumes:
@@ -42,6 +42,27 @@ services:
       - immich-network
     restart: always
 
+  immich-machine-learning:
+    image: immich-machine-learning:staging
+    build:
+      context: ../machine-learning
+      dockerfile: Dockerfile
+    entrypoint: ["/bin/sh", "./entrypoint.sh"]
+    expose:
+      - "3001"
+    volumes:
+      - ../machine-learning:/usr/src/app
+      - ${UPLOAD_LOCATION}:/usr/src/app/upload
+      - /usr/src/app/node_modules
+    env_file:
+      - .env
+    environment:
+      - NODE_ENV=development
+    depends_on:
+      - database
+    networks:
+      - immich-network
+
   immich-web:
     image: immich-web-staging:latest
     entrypoint: ["/bin/sh", "./entrypoint.sh"]
@@ -57,14 +78,12 @@ services:
       - immich-network
     restart: always
 
-
   redis:
     container_name: immich_redis
     image: redis:6.2
     networks:
       - immich-network
     restart: always
-
 
   database:
     container_name: immich_postgres
@@ -102,4 +121,4 @@ services:
 networks:
   immich-network:
 volumes:
-  pgdata:
+  pgdata:

docker/docker-compose.yml

@@ -3,7 +3,7 @@ version: "3.8"
 services:
   immich-server:
     image: altran1502/immich-server:latest
-    entrypoint: ["/bin/sh", "./entrypoint.sh"]
+    entrypoint: ["/bin/sh", "./start-server.sh"]
     expose:
       - "3000"
     volumes:
@@ -20,7 +20,23 @@ services:
     restart: always
 
   immich-microservices:
-    image: altran1502/immich-microservices:latest
+    image: altran1502/immich-server:latest
+    entrypoint: ["/bin/sh", "./start-microservices.sh"]
+    volumes:
+      - ${UPLOAD_LOCATION}:/usr/src/app/upload
+    env_file:
+      - .env
+    environment:
+      - NODE_ENV=production
+    depends_on:
+      - redis
+      - database
+    networks:
+      - immich-network
+    restart: always
+
+  immich-machine-learning:
+    image: altran1502/immich-machine-learning:latest
     entrypoint: ["/bin/sh", "./entrypoint.sh"]
     expose:
       - "3001"
@@ -47,14 +63,12 @@ services:
       - immich-network
     restart: always
 
-
   redis:
     container_name: immich_redis
     image: redis:6.2
     networks:
       - immich-network
     restart: always
-
 
   database:
     container_name: immich_postgres
@@ -73,7 +87,7 @@ services:
     networks:
       - immich-network
     restart: always
-    
+
   nginx:
     container_name: proxy_nginx
     image: nginx:latest
@@ -93,4 +107,4 @@ services:
 networks:
   immich-network:
 volumes:
-  pgdata:
+  pgdata: