Optionalizing logging setup

Signed-off-by: Lucas ONeil <[email protected]>

charts/traction/templates/ui/configmap.yaml

@@ -19,7 +19,7 @@ data:
   FRONTEND_SESSION_TIMEOUT_SECONDS: {{ .Values.ui.oidc.session.timeoutSeconds | quote }}
   FRONTEND_SESSION_COUNTDOWN_SECONDS: {{ .Values.ui.oidc.session.countdownSeconds | quote }}
   FRONTEND_TENANT_PROXY_URL: https://{{ include "tenant_proxy.host" . }}
-  FRONTEND_LOG_STREAM_URL: wss://{{ include "tenant-ui.fullname" . }}:{{ .Values.ui.service.httpPort }}/logStream
+  # FRONTEND_LOG_STREAM_URL: wss://{{ include "tenant-ui.fullname" . }}:{{ .Values.ui.service.httpPort }}/logStream
   SERVER_OIDC_JWKS: {{ .Values.ui.oidc.jwksUri | quote }}
   SERVER_OIDC_REALM: {{ .Values.ui.oidc.realm | quote }}
   SERVER_OIDC_ROLE: {{ .Values.ui.oidc.roleName | quote }}
@@ -30,7 +30,7 @@ data:
   SERVER_SMTP_SECURE: {{ .Values.ui.smtp.secure | quote }}
   SERVER_SMTP_USER: {{ .Values.ui.smtp.user | quote }}
   SERVER_TRACTION_URL: http://{{ include "tenant_proxy.fullname" . }}:{{ .Values.tenant_proxy.service.port }}
-  SERVER_LOKI_URL: http://{{ include "loki.fullname" . }}:{{ .Values.loki.service.port }}
+  SERVER_LOKI_URL: {{ .Values.ui.lokiUrl | quote }}
   UX_APP_INNKEEPER_TITLE: {{ .Values.ui.ux.appInnkeeperTitle | quote }}
   UX_APP_TITLE: {{ .Values.ui.ux.appTitle | quote }}
   UX_COPYRIGHT: {{ .Values.ui.ux.copyright | quote }}

charts/traction/values.yaml

@@ -591,18 +591,6 @@ tenant_proxy:
   ##
   tolerations: []
 
-## @section Loki configuration
-##
-loki:
-  ## Loki service configuration
-  ##
-  service:
-    ## @param loki.service.type Kubernetes Service type
-    ##
-    type: ClusterIP
-    ## @param loki.service.port Port to expose for http services
-    port: 3100
-
 ## @section Tenant-UI Configuration
 ##
 ui:
@@ -615,6 +603,8 @@ ui:
   ## @param ui.requireEmailForReservation Whether the Email field is needed for a tenant reservation
   ## If false will default [email protected] to API 
   requireEmailForReservation: true
+  ## @param ui.lokiUrl The endpoint to stream loki logs from for the Tenant UI
+  lokiUrl: ""
   image:
     ## @param ui.image.repository
     repository: ghcr.io/bcgov/traction-tenant-ui

scripts/.env-example

@@ -178,9 +178,9 @@ TENANT_UI_PORT=5101
 # which traction api are we using?
 # naming convention follows what is used by the tenant ui config
 SERVER_TRACTION_URL=http://localhost:8032
-SERVER_LOKI_URL=ws://localhost:3100
+SERVER_LOKI_URL=
 FRONTEND_TENANT_PROXY_URL=http://localhost:8032
-FRONTEND_LOG_STREAM_URL=ws://localhost:5101/logStream
+FRONTEND_LOG_STREAM_URL=
 IMAGE_BUILDTIME=
 IMAGE_TAG=scripts_tenant-ui:latest
 IMAGE_VERSION=latest

scripts/README.md

@@ -63,6 +63,8 @@ The default configuration will stand up the following environment:
 - BCovrin Test ledger... see `ACAPY_GENESIS_URL` environment variable ([http://test.bcovrin.vonx.io/genesis](http://test.bcovrin.vonx.io/genesis)).
 - previously registered Endorser DID... see `ACAPY_ENDORSER_PUBLIC_DID` environment variable.
 
+### Log streaming
+The Traction Tenant UI can optionally be configured to stream Tenant logs via a specified Loki endpoint. To see details on testing this with the local Docker setup see here.
 
 ## Run Local Traction
 - docker
@@ -203,4 +205,37 @@ You can use the wallet id and key to retrieve a token and use the Tenant API.
 9. In the bottom `AuthorizationHeader (apiKey)` section, for the Value field, enter `Bearer <your token value>` and Authorize.
 10. You are now logged in as your tenant/wallet/agent.
 11. Scroll to [GET /tenant](http://localhost:8032/api/doc#/traction-tenant/get_tenant), expand, Try it out and Execute.
-12. These are your tenant's details. Only you are authorized to fetch your tenant data.
+12. These are your tenant's details. Only you are authorized to fetch your tenant data.
+
+## Optional Local Log Streaming Setup
+The local development environment can optionally be set up to stream Tenant logs from the Traction services to the Tenant UI using Grafana Promtail and Loki.
+
+This requires some additional infrastructure that can be stood up in the docker compose environment to use locally. (in a operational deployment of Traction you would likely just need to specify your own Loki url to the Tenant UI environment, and set up the infrastructure as desired in your archetecture). By defuault these logging services will not build, but you can enable them as follows.
+
+**Ensure Loki driver is enabled in your Docker environment**  
+Can set up if needed with `docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions`. Traction assumes 'loki' alias for this.
+
+**Set environment**  
+In your local .env modify the following values
+```
+FRONTEND_LOG_STREAM_URL=ws://localhost:5101/logStream
+SERVER_LOKI_URL=ws://localhost:3100
+```
+
+For operational setup of the Tenant UI when deploying with Helm, 
+the Tenant UI configmap.yaml can be set with the following to pull the correct endpoint for the frontend:
+`FRONTEND_LOG_STREAM_URL: wss://{{ include "tenant-ui.fullname" . }}:{{ .Values.ui.service.httpPort }}/logStream`
+
+
+
+**Start up additional logging services**  
+In `/scripts` instead of just running the single docker compose, add on the logging one:  
+```
+docker compose -f docker-compose.logs.yml -f docker-compose.yml up
+```
+
+If using the `manage` script run
+```
+./manage build loki  
+./manage start loki  
+```

scripts/docker-compose.logs.yml

@@ -57,6 +57,35 @@ services:
       - "3000:3000"
     networks:
       - loki
+
+  # Override the logging config for the services in the main docker compose
+  ngrok-traction-agent:
+    logging: *default-logging
+
+  traction-acapy-image-builder:
+    logging: *default-logging
+
+  traction-agent:
+    logging: *default-logging
+
+  traction-db:
+    logging: *default-logging
+
+  tenant-ui:
+    logging: *default-logging
+
+  tenant-proxy:
+    logging: *default-logging 
+
+  endorser-agent:
+    logging: *default-logging
+
+  endorser-db:
+    logging: *default-logging
+
+  endorser-agent-1:
+    logging: *default-logging
+
 
 networks:
   loki:

scripts/docker-compose.yml

@@ -1,15 +1,8 @@
 version: "3.12"
-x-logging: &default-logging
-  driver: loki
-  options:
-    loki-url: http://localhost:3100/loki/api/v1/push
-    loki-retries: 5
-    loki-batch-size: 400
 
 services:
   ngrok-traction-agent:
     image: ngrok/ngrok
-    logging: *default-logging
     environment:
       - NGROK_AUTHTOKEN=${NGROK_AUTHTOKEN}
       - TRACTION_ACAPY_HTTP_PORT=${TRACTION_ACAPY_HTTP_PORT}
@@ -19,7 +12,6 @@ services:
 
   traction-acapy-image-builder:
     pull_policy: missing
-    logging: *default-logging
     build:
       context: ../plugins
       dockerfile: ./docker/Dockerfile
@@ -33,7 +25,6 @@ services:
       context: ../services/aca-py
       dockerfile: Dockerfile.acapy
     image: traction:traction-agent
-    logging: *default-logging
     depends_on:
       traction-acapy-image-builder:
         condition: service_started
@@ -105,7 +96,6 @@ services:
 
   traction-db:
     image: "postgres:14.5"
-    logging: *default-logging
     environment:
       - POSTGRES_USER=${POSTGRESQL_USER}
       - POSTGRES_PASSWORD=${POSTGRESQL_PASSWORD}
@@ -124,7 +114,6 @@ services:
 
   tenant-ui:
     pull_policy: missing
-    logging: *default-logging
     build:
       context: ../services/tenant-ui
       dockerfile: Dockerfile
@@ -160,7 +149,6 @@ services:
 
   tenant-proxy:
     pull_policy: missing
-    logging: *default-logging
     build:
       context: ../plugins
       dockerfile: ./docker/Dockerfile.tenant-proxy
@@ -182,7 +170,6 @@ services:
     depends_on:
       endorser-db:
         condition: service_healthy
-    logging: *default-logging
     ports:
       - ${ACAPY_ENDORSER_ADMIN_PORT}:${ACAPY_ENDORSER_ADMIN_PORT}
       - ${ACAPY_ENDORSER_HTTP_PORT}:${ACAPY_ENDORSER_HTTP_PORT}
@@ -221,7 +208,6 @@ services:
     depends_on:
       endorser-db:
         condition: service_healthy
-    logging: *default-logging
     ports:
       - ${ACAPY_ENDORSER_1_ADMIN_PORT}:${ACAPY_ENDORSER_1_ADMIN_PORT}
       - ${ACAPY_ENDORSER_1_HTTP_PORT}:${ACAPY_ENDORSER_1_HTTP_PORT}
@@ -257,7 +243,6 @@ services:
 
   endorser-db:
     image: "postgres:14.5"
-    logging: *default-logging
     environment:
       - POSTGRES_USER=${POSTGRESQL_USER}
       - POSTGRES_PASSWORD=${POSTGRESQL_PASSWORD}

scripts/manage

@@ -24,7 +24,12 @@ generate_traction_ascii_art() {
 
 # Global variable to hold Docker Compose command
 compose_cmd=""
-compose_files_opt="-f docker-compose.logs.yml -f docker-compose.yml"
+compose_files_opt=""
+include_loki_services=false
+if [ "$2" == "loki" ]; then
+  include_loki_services=true
+  compose_files_opt="-f docker-compose.logs.yml -f docker-compose.yml"
+fi
 
 # Function to determine the correct Docker Compose command
 get_docker_compose_command() {
@@ -66,7 +71,12 @@ build_docker_images() {
   fi
 
   built_services="traction-acapy-image-builder tenant-ui tenant-proxy endorser-api endorser-api-1"
-  pulled_services="loki promtail grafana ngrok-traction-agent traction-db endorser-db endorser-db-1 maildev endorser-agent endorser-agent-1"
+  if [ "$include_loki_services" == true ]; then
+    pulled_services="loki promtail grafana ngrok-traction-agent traction-db endorser-db endorser-db-1 maildev endorser-agent endorser-agent-1"
+  else
+    pulled_services="ngrok-traction-agent traction-db endorser-db endorser-db-1 maildev endorser-agent endorser-agent-1"
+  fi
+
   $compose_cmd build $built_services --no-cache --parallel
   $compose_cmd pull $pulled_services
 }

services/tenant-ui/README.md

@@ -146,3 +146,7 @@ If no value is set using either of these methods MATOMO tracker code will never
 
 For more information on configuration settings see
 [Set up your configuration](https://github.com/bcgov/traction/tree/main/services/tenant-ui#set-up-your-configuration)
+
+## Log Streaming
+
+The Tenant UI can display streamed logs from a Loki aggregator. For details on setup, see ["Optional Local Log Streaming Setup" in the scripts README](../../scripts/README.md)

services/tenant-ui/frontend/src/components/layout/Sidebar.vue

@@ -15,13 +15,12 @@
 </template>
 
 <script setup lang="ts">
-import { ref } from 'vue';
 import PanelMenu from 'primevue/panelmenu';
+import PanelMenuItemLink from '../common/PanelMenuItemLink.vue';
 import ProgressSpinner from 'primevue/progressspinner';
 import { storeToRefs } from 'pinia';
 import { useI18n } from 'vue-i18n';
 import { useConfigStore, useTenantStore } from '../../store';
-import PanelMenuItemLink from '../common/PanelMenuItemLink.vue';
 
 const { t } = useI18n();
 const { config } = useConfigStore();

services/tenant-ui/frontend/src/components/layout/innkeeper/Sidebar.vue

@@ -13,15 +13,16 @@
 </template>
 
 <script setup lang="ts">
-import { ref } from 'vue';
 import PanelMenu from 'primevue/panelmenu';
 import { useI18n } from 'vue-i18n';
 import PanelMenuItemLink from '@/components/common/PanelMenuItemLink.vue';
+import { useConfigStore } from '../../../store';
 
 const { t } = useI18n();
+const { config } = useConfigStore();
 
 const ROOT = '/innkeeper/';
-const items = ref([
+const items = [
   {
     label: t('reservations.reservations'),
     icon: 'pi pi-fw pi-book',
@@ -59,12 +60,15 @@ const items = ref([
     icon: 'pi pi-fw pi-question-circle',
     route: ROOT + 'about',
   },
-  {
+];
+
+if (config?.frontend?.logStreamUrl) {
+  items.push({
     label: t('log.log'),
     icon: 'pi pi-fw pi-file',
     route: ROOT + 'log',
-  },
-]);
+  });
+}
 </script>
 <style scoped lang="scss">
 .sidebar-app-title.small {

services/tenant-ui/frontend/src/main.ts

@@ -69,7 +69,7 @@ async function loadApp() {
 
   // Log Setup
   if (!configStore?.config?.frontend?.logStreamUrl) {
-    console.warn('Log not configured');
+    console.log('Log streaming not configured');
     router.removeRoute('Log');
   }
 

services/tenant-ui/src/index.ts

@@ -7,8 +7,9 @@ import { router } from "./routes/router";
 
 import { configureLogStream } from "./services/log-stream";
 
-const PORT: number = parseInt(config.get("server.port") as string, 10);
 const API_ROOT: string = config.get("server.apiPath");
+const LOKI_URL: string = config.get("server.lokiUrl");
+const PORT: number = parseInt(config.get("server.port") as string, 10);
 const STATIC_FILES_PATH: string = config.get("server.staticFiles");
 
 import history from "connect-history-api-fallback";
@@ -50,8 +51,10 @@ app.use("/config", (_, res, next) => {
 // This service's api endpoints
 app.use(API_ROOT, router);
 
-configureLogStream(
-  app.listen(PORT, () => {
-    console.log(`Listening on port ${PORT}, apiroot: ${API_ROOT}`);
-  })
-);
+if (LOKI_URL) {
+  configureLogStream(
+    app.listen(PORT, () => {
+      console.log(`Listening on port ${PORT}, apiroot: ${API_ROOT}`);
+    })
+  );
+}