[DO NOT MERGE!] Separate docker compose watch config and fix client container when ru…

Dockerfile

@@ -14,7 +14,12 @@
 #       "-component" "admin_ui" \
 #       "-config" "/configs/grr.server.yaml"
 #
-# - Run the grr client component:
+# - Run the grr client component via repacking client templates:
+#   Client installers for different operating systems are created by
+#   repacking client templates, which are included in the grr docker image.
+#   (they are currently only build in the github workflow that creates the
+#   grr docker image). To create these client installers and run a container
+#   with the GRR client follow these steps:
 #   -- Start the container and mount the client config directory:
 #       $ docker run -it \
 #          -v $(pwd)/docker_config_files/client:/configs \
@@ -35,6 +40,8 @@
 #             ...        COMMAND
 #             ...        fleetspeak-client -config /configs/client.config
 #             ...        python -m grr_response_client.client ...
+# - To run a GRR client container without repacking checkout out the
+#   Dockerfile.client file. 
 
 FROM ubuntu:22.04
 

Dockerfile.client

@@ -0,0 +1,64 @@
+### A Docker image containing fleetspeak and grr clients.
+#
+# Fleetspeak client starts grr client as a subprocess based on the config.
+# 
+# Fleetspeak client requires connectivity to fleetspeak server, we
+# recommend running this client in the docker compose stack or the config
+# needs to be adjusted.
+#
+# See documentation in compose.watch.yaml on how to start the Compose stack.
+#
+# (Optional) To verify if the client runs, check if the fleetspeak and
+# grr processes are running inside the container.
+#   - Open a shell in the container:
+#       $ docker exec -it grr-client /bin/bash
+#   - Check the running processes:
+#    $ ps aux
+#      ...
+#      ...  fleetspeak-client -config /configs/client/client.config
+#      ...  /bin/bash /configs/client/grr_fleetspeak_client.sh --config /configs/client/grr.client.yaml
+#      ...  /usr/share/grr-server/bin/python /usr/share/grr-server/bin/grr_fleetspeak_client --config /configs/client/grr.client.yaml
+#      ...
+FROM ubuntu:22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Buffering output (sometimes indefinitely if a thread is stuck in
+# a loop) makes for a non-optimal user experience when containers
+# are run in the foreground, so we disable that.
+ENV PYTHONUNBUFFERED=0
+
+RUN apt-get update && \
+  apt-get install -y \
+  python-is-python3 \
+  python3-dev \
+  python3-pip \
+  python3-venv \
+  build-essential \
+  linux-headers-generic
+
+ENV FLEETSPEAK_BIN /fleetspeak/bin
+RUN mkdir -p $FLEETSPEAK_BIN
+COPY --from=ghcr.io/google/fleetspeak:latest /fleetspeak/bin/client $FLEETSPEAK_BIN/fleetspeak-client
+ENV PATH=${FLEETSPEAK_BIN}:${PATH}
+
+ENV VIRTUAL_ENV=/usr/share/grr-server
+ENV GRR_SOURCE=/usr/src/grr
+
+RUN python -m venv --system-site-packages $VIRTUAL_ENV
+ENV PATH=${VIRTUAL_ENV}/bin:${PATH}
+
+RUN mkdir -p ${GRR_SOURCE}
+ADD . ${GRR_SOURCE}
+
+WORKDIR ${GRR_SOURCE}
+
+RUN ${VIRTUAL_ENV}/bin/python -m pip install \
+  -e grr/proto \
+  -e grr/core \
+  -e grr/client
+
+RUN ${VIRTUAL_ENV}/bin/python grr/proto/makefile.py && \
+  ${VIRTUAL_ENV}/bin/python grr/core/grr_response_core/artifacts/makefile.py
+
+ENTRYPOINT [ "fleetspeak-client" ]

compose.testing.yaml

@@ -11,3 +11,4 @@ services:
 
   grr-client:
     image: ghcr.io/google/grr:testing
+    privileged: true

compose.watch.yaml

@@ -0,0 +1,75 @@
+# Run grr in watch mode with:
+# ```
+#   $ docker compose -f compose.yaml -f compose.watch.yaml watch
+# ```
+#
+# This is merging compose.watch.yaml with compose.yaml
+# (https://docs.docker.com/reference/compose-file/merge/) to apply adjustments
+# for running containers from source code:
+#   - Add `build: ...` to build images from local Dockerfiles.
+#   - Update `image: ... ` to prevent overwriting image tags fetched from github
+#     container registry.
+#   - Remove the admin ui healthcheck and it's dependency in the grr-client.
+#     The healthcheck indicated if the client templates were repacked into
+#     installers, as the client is installed from the debian installer in the
+#     default setup. But here we run the client from source.
+#     (The client templates are also not available in the locally build image,
+#     they are build in the github workflow.)
+#   - Update the grr-client entrypoint to start the client directly from source
+#     instead of first installing the debian installer.
+#   - Added `develop: watch: ...` to trigger a container sync and restart
+#     when the code changes.
+services:
+  grr-admin-ui:
+    build: .
+    image: watch-grr-admin-ui
+    healthcheck:
+      test: ""
+    develop:
+      watch:
+        - action: sync+restart
+          path: ./grr
+          target: /usr/src/grr/grr
+          ignore:
+            - client/
+
+  grr-client:
+    build:
+      context: .
+      dockerfile: ./Dockerfile.client
+    image: watch-grr-client
+    entrypoint: [
+      "/bin/bash",
+      "-c",
+      "fleetspeak-client -config /configs/client/client.config"
+    ]
+    develop:
+      watch:
+        - action: sync+restart
+          path: ./grr
+          target: /usr/src/grr/grr
+          ignore:
+            - server/
+
+  grr-fleetspeak-frontend:
+    build: .
+    image: watch-grr-fleetspeak-frontend
+    develop:
+      watch:
+        - action: sync+restart
+          path: ./grr
+          target: /usr/src/grr/grr
+          ignore:
+            - client/
+
+  grr-worker:
+    build: .
+    image: watch-grr-worker
+    develop:
+      watch:
+        - action: sync+restart
+          path: ./grr
+          target: /usr/src/grr/grr
+          ignore:
+            - client/
+

compose.yaml

@@ -1,15 +1,16 @@
 version: "3.8"
 services:
   db:
-    image: mysql:8.2
+    image: mysql:8.4
     env_file: docker_config_files/mysql/.env
     container_name: grr-db
     hostname: mysql-host
     command: [
       --max_allowed_packet=40M,
       --log_bin_trust_function_creators=1,
       --innodb_redo_log_capacity=167772160,
-      --innodb_log_file_size=2500M
+      --innodb_log_file_size=2500M,
+      --restrict-fk-on-non-standard-key=OFF
     ]
     restart: always
     volumes:
@@ -27,7 +28,6 @@ services:
       retries: 10
 
   grr-admin-ui:
-    build: .
     image: ghcr.io/google/grr:latest
     container_name: grr-admin-ui
     hostname: admin-ui
@@ -58,16 +58,9 @@ services:
       test: "/configs/healthchecks/grr-admin-ui.sh"
       timeout: 10s
       retries: 10
-    develop:
-      watch:
-        - action: sync+restart
-          path: ./grr
-          target: /usr/src/grr/grr
-          ignore:
-            - client/
+
 
   grr-fleetspeak-frontend:
-    build: .
     image: ghcr.io/google/grr:latest
     container_name: grr-fleetspeak-frontend
     hostname: grr-fleetspeak-frontend
@@ -87,13 +80,6 @@ services:
       - -config
       - /configs/server/grr.server.yaml
       - --verbose
-    develop:
-      watch:
-        - action: sync+restart
-          path: ./grr
-          target: /usr/src/grr/grr
-          ignore:
-            - client/
 
   fleetspeak-admin:
     image: ghcr.io/google/fleetspeak:latest
@@ -145,7 +131,6 @@ services:
     ]
 
   grr-worker:
-    build: .
     image: ghcr.io/google/grr:latest
     container_name: grr-worker
     volumes:
@@ -163,16 +148,8 @@ services:
       - -config
       - /configs/server/grr.server.yaml
       - --verbose
-    develop:
-      watch:
-        - action: sync+restart
-          path: ./grr
-          target: /usr/src/grr/grr
-          ignore:
-            - client/
 
   grr-client:
-    build: .
     image: ubuntu:22.04
     container_name: grr-client
     depends_on:
@@ -202,13 +179,6 @@ services:
       test: "/configs/healthchecks/grr-client.sh"
       timeout: 10s
       retries: 10
-    develop:
-      watch:
-        - action: sync+restart
-          path: ./grr
-          target: /usr/src/grr/grr
-          ignore:
-            - server/
 
 volumes:
   db_data: