Merge pull request #130 from Calcagiara/dhubDremioUserTestChangesKfpB…

…ucket feat(digitalhub): dremio now uses coder email and an automatically generated password; core tests now take the token from core directly; kubeflow-pipelines now uses a separate bucket
scc-digitalhub · Oct 18, 2024 · a79cde2 · a79cde2
2 parents 599818a + 3395b88
commit a79cde2
Show file tree

Hide file tree

Showing 8 changed files with 96 additions and 75 deletions.
diff --git a/charts/digitalhub/Chart.yaml b/charts/digitalhub/Chart.yaml
@@ -7,7 +7,7 @@ maintainers:
     url: https://github.com/ffais
   - name: calcagiara
     url: https://github.com/Calcagiara
-version: "0.8.0-beta2"
+version: "0.8.0-beta3"
 appVersion: "0.8.0"
 dependencies:
 - name: apigw-operator

diff --git a/charts/digitalhub/confs/coder/dremio-main.tf b/charts/digitalhub/confs/coder/dremio-main.tf
@@ -105,19 +105,6 @@ variable "minio_digitalhub_user_secret" {
   type = string
 }
 
-data "coder_parameter" "admin_password" {
-  name         = "admin_password"
-  display_name = "Dremio Admin Password"
-  description  = "Choose a password for Dremio admin account must be at least 8 letters long, must contain at least one number and one letter"
-  type         = "string"
-  icon         = "/emojis/1f510.png"
-  mutable      = false
-  # validation {
-  #   regex = "[a-zA-Z][0-9][a-zA-Z0-9]{6,}|[a-zA-Z]{2}[0-9][a-zA-Z0-9]{5,}|[a-zA-Z]{3}[0-9][a-zA-Z0-9]{4,}|[a-zA-Z]{5}[0-9][a-zA-Z0-9]{3,}|[a-zA-Z]{6}[0-9][a-zA-Z0-9]{2,}|[a-zA-Z]{7,}[0-9][a-zA-Z0-9]*|[0-9][a-zA-Z][a-zA-Z0-9]{6,}|[0-9]{2}[a-zA-Z][a-zA-Z0-9]{5,}|[0-9]{3}[a-zA-Z][a-zA-Z0-9]{4,}|[0-9]{5}[a-zA-Z][a-zA-Z0-9]{3,}|[0-9]{6}[a-zA-Z][a-zA-Z0-9]{2,}|[0-9]{7,}[a-zA-Z][a-zA-Z0-9]*"
-  #   error = "Invalid password: must be at least 8 letters long, must contain at least one number and one letter"
-  # }
-}
-
 provider "kubernetes" {
   # Authenticate via ~/.kube/config or a Coder-specific ServiceAccount, depending on admin preferences
   config_path = var.use_kubeconfig == true ? "~/.kube/config" : null
@@ -127,6 +114,12 @@ data "coder_workspace" "me" {}
 
 data "coder_workspace_owner" "me" {}
 
+resource "random_password" "password" {
+  length           = 16
+  special          = true
+  override_special = "!%&*()-_=+:?"
+}
+
 resource "coder_agent" "dremio" {
   os             = "linux"
   arch           = "amd64"
@@ -199,6 +192,16 @@ resource "coder_metadata" "dremio" {
     key   = "URL"
     value = local.dremio_url
   }
+  item {
+  key = "Username"
+  value = data.coder_workspace_owner.me.email
+  sensitive = false
+  }
+  item {
+  key = "Password"
+  value = random_password.password.result
+  sensitive = true
+  }
 }
 
 resource "kubernetes_persistent_volume_claim" "dremio-data" {
@@ -313,13 +316,40 @@ resource "kubernetes_job" "source-init" {
           image   = "badouralix/curl-jq"
           command = ["/bin/sh", "-c", "until [ \"$(curl -s -w '%%{http_code}' -o /dev/null \"http://${kubernetes_service.dremio-service.metadata.0.name}:9047/api/v2/buildinfo\")\" -eq 200 ]; do echo \"waiting for dremio to be ready\"; sleep 5; done"]
         }
+        init_container {
+          name              = "init-dremio-data"
+          image             = "dremio/dremio-oss:24.1.0"
+          image_pull_policy = "IfNotPresent"
+          command           = ["/bin/bash", "/tmp/init/init-data.sh"]
+          env {
+            name  = "ADMIN_PASSWORD"
+            value = random_password.password.result
+          }
+          env {
+            name  = "DREMIO_CODER_EMAIL"
+            value = data.coder_workspace_owner.me.email
+          }
+          env {
+            name  = "DREMIO_URL"
+            value = kubernetes_service.dremio-service.metadata.0.name
+          }
+          volume_mount {
+            mount_path = "/tmp/init/"
+            name       = "dremio-init-data"
+            read_only  = false
+          }
+          security_context {
+            run_as_user                = "999"
+            allow_privilege_escalation = false
+          }
+        }
         container {
           name    = "dremio-add-sources"
           image   = "badouralix/curl-jq"
           command = ["/bin/sh", "/init-files/add_source_with_api.sh"]
           env {
             name  = "ADMIN_PASSWORD"
-            value = data.coder_parameter.admin_password.value
+            value = random_password.password.result
           }
           env {
             name  = "DREMIO_URL"
@@ -377,6 +407,10 @@ resource "kubernetes_job" "source-init" {
               }
             }
           }
+          env {
+            name  = "DREMIO_CODER_EMAIL"
+            value = data.coder_workspace_owner.me.email
+          }
           volume_mount {
             name       = "init-files"
             mount_path = "/init-files"
@@ -393,6 +427,17 @@ resource "kubernetes_job" "source-init" {
             }
           }
         }
+        volume {
+          name = "dremio-init-data"
+          config_map {
+            name = "dremio-init-data"
+            items {
+              key  = "init-data.sh"
+              path = "init-data.sh"
+            }
+          }
+
+        }
       }
     }
   }
@@ -452,30 +497,6 @@ resource "kubernetes_deployment" "dremio" {
           fs_group     = "999"
           run_as_group = "999"
         }
-        init_container {
-          name              = "init-dremio-data"
-          image             = "dremio/dremio-oss:24.1.0"
-          image_pull_policy = "IfNotPresent"
-          command           = ["/bin/bash", "/tmp/init/init-data.sh"]
-          env {
-            name  = "ADMIN_PASSWORD"
-            value = data.coder_parameter.admin_password.value
-          }
-          volume_mount {
-            mount_path = "/opt/dremio/data"
-            name       = "dremio-data"
-            read_only  = false
-          }
-          volume_mount {
-            mount_path = "/tmp/init/"
-            name       = "dremio-init-data"
-            read_only  = false
-          }
-          security_context {
-            run_as_user                = "999"
-            allow_privilege_escalation = false
-          }
-        }
         container {
           name              = "dremio"
           image             = var.image
@@ -518,21 +539,6 @@ resource "kubernetes_deployment" "dremio" {
             read_only  = false
           }
         }
-        volume {
-          name = "dremio-init-data"
-          config_map {
-            name = "dremio-init-data"
-            items {
-              key  = "dremio-backup.tar"
-              path = "dremio-backup.tar"
-            }
-            items {
-              key  = "init-data.sh"
-              path = "init-data.sh"
-            }
-          }
-
-        }
         volume {
           name = "dremio-home"
           empty_dir {}

diff --git a/charts/digitalhub/confs/dremio/add_source_with_api.sh b/charts/digitalhub/confs/dremio/add_source_with_api.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-TOKEN=$(curl -s -X POST "http://${DREMIO_URL}:9047/apiv2/login" --header 'Content-Type: application/json' --data-raw "{\"userName\": \"admin\",\"password\": \"${ADMIN_PASSWORD}\"}" | jq -r .token)
+TOKEN=$(curl -s -X POST "http://${DREMIO_URL}:9047/apiv2/login" --header 'Content-Type: application/json' --data-raw "{\"userName\": \"${DREMIO_CODER_EMAIL}\",\"password\": \"${ADMIN_PASSWORD}\"}" | jq -r .token)
 
 echo "add postgres"
 curl -v -s -X POST "http://${DREMIO_URL}:9047/api/v3/catalog" \

diff --git a/charts/digitalhub/confs/dremio/init-data.sh b/charts/digitalhub/confs/dremio/init-data.sh
@@ -1,12 +1,11 @@
 #!/usr/bin/env bash
-DIR="/opt/dremio/data/db"
-if [ -d "$DIR" ]; then
-    echo "directory already exist"
-    exit 0
-else
-    mkdir -p /opt/dremio/data/db
-    tar xvf /tmp/init/dremio-backup.tar --directory /tmp
-    /opt/dremio/bin/dremio-admin restore -d /tmp/dremio_backup_2023-07-06_12.02/
-    echo "$ADMIN_PASSWORD"
-    /opt/dremio/bin/dremio-admin set-password -u admin -p "$ADMIN_PASSWORD"
-fi
+
+curl "http://${DREMIO_URL}:9047/apiv2/bootstrap/firstuser" -X PUT \
+      -H 'Authorization: _dremionull' -H 'Content-Type: application/json' \
+      -d '{
+         "userName": "'"$DREMIO_CODER_EMAIL"'",
+         "password": "'"$ADMIN_PASSWORD"'",
+         "firstName": "digitalhub",
+         "lastName": "digitalhub",
+         "email": "'"$DREMIO_CODER_EMAIL"'"
+       }'
diff --git a/charts/digitalhub/confs/tests/python-test/python-test-container.py b/charts/digitalhub/confs/tests/python-test/python-test-container.py
@@ -40,7 +40,7 @@ def main():
         scope = 'tenant1-core'
         client = BackendApplicationClient(client_id=client_id)
         oauth = OAuth2Session(client=client, scope=scope)
-        token = oauth.fetch_token(token_url='https://aac.digitalhub-dev.smartcommunitylab.it/oauth/token', client_id=client_id, client_secret=client_secret, scope=scope)
+        token = oauth.fetch_token(token_url='https://core.tenant1.digitalhub-dev.smartcommunitylab.it/auth/token', client_id=client_id, client_secret=client_secret, scope=scope)
         os.environ["DHCORE_ACCESS_TOKEN"] = token["access_token"]
 
     # Get or create project

diff --git a/charts/digitalhub/confs/tests/python-test/python-test-dbt.py b/charts/digitalhub/confs/tests/python-test/python-test-dbt.py
@@ -40,7 +40,7 @@ def main():
         scope = 'tenant1-core'
         client = BackendApplicationClient(client_id=client_id)
         oauth = OAuth2Session(client=client, scope=scope)
-        token = oauth.fetch_token(token_url='https://aac.digitalhub-dev.smartcommunitylab.it/oauth/token', client_id=client_id, client_secret=client_secret, scope=scope)
+        token = oauth.fetch_token(token_url='https://core.tenant1.digitalhub-dev.smartcommunitylab.it/auth/token', client_id=client_id, client_secret=client_secret, scope=scope)
         os.environ["DHCORE_ACCESS_TOKEN"] = token["access_token"]
 
     # Get or create project

diff --git a/charts/digitalhub/templates/tests/python-test.yaml b/charts/digitalhub/templates/tests/python-test.yaml
@@ -36,13 +36,13 @@ spec:
             - name: "CORE_CLIENT_ID"
               valueFrom:
                 secretKeyRef:
-                  name: "aac-digitalhub-tenant1-core-secret"
-                  key: "clientid"
+                  name: "core-auth-creds"
+                  key: "clientId"
             - name: "CORE_CLIENT_SECRET"
               valueFrom:
                 secretKeyRef:
-                  name: "aac-digitalhub-tenant1-core-secret"
-                  key: "clientsecret"
+                  name: "core-auth-creds"
+                  key: "clientSecret"
             {{- if $.Values.core.ingress.enabled }}
             {{- with (index $.Values.core.ingress.hosts 0) }}
             - name: "DHCORE_ENDPOINT"

diff --git a/charts/digitalhub/values.yaml b/charts/digitalhub/values.yaml
@@ -28,9 +28,12 @@ global:
     digitalhubUserSecret: digitalhub-minio-creds
     digitalhubUser: &digitalhubUser digitalhub
     digitalhubPassword: &digitalhubPassword digitalhub
+    kfpUser: &kfpUser kfp
+    kfpPassword: &kfpPassword kfpwd123
     endpoint: &minioEndpoint "minio"
     endpointPort: &minioEndpointPort "9000"
     bucket: &minioBucket "datalake"
+    kfpBucket: &kfpBucket "kfp"
     protocol: &minioProtocol "http"
   solr:
     fullNameOverride: &solrFullNameOverride digitalhub
@@ -434,6 +437,9 @@ minio:
     - name: datalake
       policy: none
       purge: false
+    - name: kfp
+      policy: none
+      purge: false
   resources:
     requests:
       memory: 0.5Gi
@@ -468,6 +474,9 @@ minio:
     - accessKey: *digitalhubUser
       secretKey: *digitalhubPassword
       policy: readwritedigitalhub
+    - accessKey: *kfpUser
+      secretKey: *kfpPassword
+      policy: readwritekfp
   policies:
     - name: readwritedigitalhub
       statements:
@@ -476,6 +485,13 @@ minio:
             - 'arn:aws:s3:::datalake/*'
           actions:
             - "s3:*"
+    - name: readwritekfp
+      statements:
+        - effect: Allow
+          resources:
+            - 'arn:aws:s3:::kfp/*'
+          actions:
+            - "s3:*"
 
 minio-operator:
   enabled: true
@@ -526,9 +542,9 @@ kubeflow-pipelines:
       nodePort: 30100
   s3:
     enabled: true
-    accessKey: *digitalhubUser
-    secretKey: *digitalhubPassword
+    accessKey: *kfpUser
+    secretKey: *kfpPassword
     endpoint: *minioEndpoint
     endpointPort: *minioEndpointPort
-    bucket: *minioBucket
+    bucket: *kfpBucket
     protocol: *minioProtocol