feat(api): Add user-configurable secrets for deployments and ensembli…

…ng jobs (#403) * Update openapi specs * Update autogenerated golang client files * Update autogenerated python client files * Add user-configured secrets to batch jobs * Add missing python client file * Fix lint comments * Add user-configured secrets to enricher and ensembler deployments * Black turing sdk files * Add db migration scripts * Fix lint comments * Add sdk changes to allow user-secrets to be mounted * Update e2e tests * Fix enricher secrets parsing * Fix broken validator test * Fix broken validator tests and add required tag to secrets field in pyfunc ensembler config * Fix e2e tests * Fix secret map key bug * Update jsonb column to have empty list as default value * Update db migration scripts * Add missing step in api server to add enricher and ensembler secrets * Update api specs and autogenerated client files * Replace isnumeric check with check that passes floats * Update react-lazylog with published version * Add panels to display secrets * Add steps to configure secrets in forms * Add new unit test for autoscaling policy * Add fix to unit tests to prevent race conditions * Update openapi specs and autogenerated files * Remove redundant variable assignment
caraml-dev · Feb 21, 2025 · 9b0ccd2 · 9b0ccd2
1 parent 1175212
commit 9b0ccd2
Show file tree

Hide file tree

Showing 76 changed files with 2,053 additions and 421 deletions.
diff --git a/api/api/openapi.bundle.yaml b/api/api/openapi.bundle.yaml
@@ -1617,6 +1617,11 @@ components:
             value: value
           - name: name
             value: value
+          secrets:
+          - mlp_secret_name: mlp_secret_name
+            env_var_name: env_var_name
+          - mlp_secret_name: mlp_secret_name
+            env_var_name: env_var_name
         monitoring_url: monitoring_url
         environment_name: environment_name
       properties:
@@ -1672,6 +1677,11 @@ components:
           value: value
         - name: name
           value: value
+        secrets:
+        - mlp_secret_name: mlp_secret_name
+          env_var_name: env_var_name
+        - mlp_secret_name: mlp_secret_name
+          env_var_name: env_var_name
       properties:
         artifact_uri:
           type: string
@@ -1680,6 +1690,10 @@ components:
         service_account_name:
           type: string
           x-go-custom-tag: validate:"required"
+        secrets:
+          items:
+            $ref: '#/components/schemas/MountedMLPSecret'
+          type: array
         resources:
           $ref: '#/components/schemas/EnsemblingResources'
         run_id:
@@ -2136,6 +2150,11 @@ components:
             value: value
           - name: name
             value: value
+          secrets:
+          - mlp_secret_name: mlp_secret_name
+            env_var_name: env_var_name
+          - mlp_secret_name: mlp_secret_name
+            env_var_name: env_var_name
           timeout: timeout
         routes:
         - endpoint: endpoint
@@ -2186,6 +2205,11 @@ components:
               value: value
             - name: name
               value: value
+            secrets:
+            - mlp_secret_name: mlp_secret_name
+              env_var_name: env_var_name
+            - mlp_secret_name: mlp_secret_name
+              env_var_name: env_var_name
             timeout: timeout
           updated_at: 2000-01-23T04:56:07.000+00:00
           standard_config:
@@ -2220,6 +2244,11 @@ components:
               value: value
             - name: name
               value: value
+            secrets:
+            - mlp_secret_name: mlp_secret_name
+              env_var_name: env_var_name
+            - mlp_secret_name: mlp_secret_name
+              env_var_name: env_var_name
             timeout: timeout
       properties:
         id:
@@ -2438,6 +2467,11 @@ components:
           value: value
         - name: name
           value: value
+        secrets:
+        - mlp_secret_name: mlp_secret_name
+          env_var_name: env_var_name
+        - mlp_secret_name: mlp_secret_name
+          env_var_name: env_var_name
         timeout: timeout
       properties:
         id:
@@ -2460,6 +2494,10 @@ components:
           items:
             $ref: '#/components/schemas/EnvVar'
           type: array
+        secrets:
+          items:
+            $ref: '#/components/schemas/MountedMLPSecret'
+          type: array
         service_account:
           description: |
             (Optional) Name of the secret registered in the current MLP project that contains the Google service account JSON key. This secret will be mounted as a file inside the container and the environment variable GOOGLE_APPLICATION_CREDENTIALS will point to the service account file."
@@ -2479,6 +2517,7 @@ components:
       - image
       - port
       - resource_request
+      - secrets
       - timeout
       type: object
     RouterEnsemblerConfig:
@@ -2500,6 +2539,11 @@ components:
             value: value
           - name: name
             value: value
+          secrets:
+          - mlp_secret_name: mlp_secret_name
+            env_var_name: env_var_name
+          - mlp_secret_name: mlp_secret_name
+            env_var_name: env_var_name
           timeout: timeout
         updated_at: 2000-01-23T04:56:07.000+00:00
         standard_config:
@@ -2534,6 +2578,11 @@ components:
             value: value
           - name: name
             value: value
+          secrets:
+          - mlp_secret_name: mlp_secret_name
+            env_var_name: env_var_name
+          - mlp_secret_name: mlp_secret_name
+            env_var_name: env_var_name
           timeout: timeout
       properties:
         id:
@@ -2609,6 +2658,11 @@ components:
           value: value
         - name: name
           value: value
+        secrets:
+        - mlp_secret_name: mlp_secret_name
+          env_var_name: env_var_name
+        - mlp_secret_name: mlp_secret_name
+          env_var_name: env_var_name
         timeout: timeout
       nullable: true
       properties:
@@ -2630,6 +2684,10 @@ components:
           items:
             $ref: '#/components/schemas/EnvVar'
           type: array
+        secrets:
+          items:
+            $ref: '#/components/schemas/MountedMLPSecret'
+          type: array
         service_account:
           description: |
             (Optional) Name of the secret registered in the current MLP project that contains the Google service account JSON key. This secret will be mounted as a file inside the container and the environment variable GOOGLE_APPLICATION_CREDENTIALS will point to the service account file."
@@ -2641,6 +2699,7 @@ components:
       - image
       - port
       - resource_request
+      - secrets
       - timeout
       type: object
     EnsemblerPyfuncConfig:
@@ -2662,6 +2721,11 @@ components:
           value: value
         - name: name
           value: value
+        secrets:
+        - mlp_secret_name: mlp_secret_name
+          env_var_name: env_var_name
+        - mlp_secret_name: mlp_secret_name
+          env_var_name: env_var_name
         timeout: timeout
       nullable: true
       properties:
@@ -2680,10 +2744,16 @@ components:
           items:
             $ref: '#/components/schemas/EnvVar'
           type: array
+        secrets:
+          items:
+            $ref: '#/components/schemas/MountedMLPSecret'
+          type: array
       required:
       - ensembler_id
+      - env
       - project_id
       - resource_request
+      - secrets
       - timeout
       type: object
     TrafficRule:
@@ -2794,6 +2864,11 @@ components:
               value: value
             - name: name
               value: value
+            secrets:
+            - mlp_secret_name: mlp_secret_name
+              env_var_name: env_var_name
+            - mlp_secret_name: mlp_secret_name
+              env_var_name: env_var_name
             timeout: timeout
           routes:
           - endpoint: endpoint
@@ -2867,6 +2942,11 @@ components:
                 value: value
               - name: name
                 value: value
+              secrets:
+              - mlp_secret_name: mlp_secret_name
+                env_var_name: env_var_name
+              - mlp_secret_name: mlp_secret_name
+                env_var_name: env_var_name
               timeout: timeout
             updated_at: 2000-01-23T04:56:07.000+00:00
             standard_config:
@@ -2901,6 +2981,11 @@ components:
                 value: value
               - name: name
                 value: value
+              secrets:
+              - mlp_secret_name: mlp_secret_name
+                env_var_name: env_var_name
+              - mlp_secret_name: mlp_secret_name
+                env_var_name: env_var_name
               timeout: timeout
           log_config:
             bigquery_config:
@@ -2957,6 +3042,11 @@ components:
             value: value
           - name: name
             value: value
+          secrets:
+          - mlp_secret_name: mlp_secret_name
+            env_var_name: env_var_name
+          - mlp_secret_name: mlp_secret_name
+            env_var_name: env_var_name
           timeout: timeout
         routes:
         - endpoint: endpoint
@@ -3030,6 +3120,11 @@ components:
               value: value
             - name: name
               value: value
+            secrets:
+            - mlp_secret_name: mlp_secret_name
+              env_var_name: env_var_name
+            - mlp_secret_name: mlp_secret_name
+              env_var_name: env_var_name
             timeout: timeout
           updated_at: 2000-01-23T04:56:07.000+00:00
           standard_config:
@@ -3064,6 +3159,11 @@ components:
               value: value
             - name: name
               value: value
+            secrets:
+            - mlp_secret_name: mlp_secret_name
+              env_var_name: env_var_name
+            - mlp_secret_name: mlp_secret_name
+              env_var_name: env_var_name
             timeout: timeout
         log_config:
           bigquery_config:
@@ -3476,6 +3576,21 @@ components:
           format: int32
           type: integer
       type: object
+    MountedMLPSecret:
+      example:
+        mlp_secret_name: mlp_secret_name
+        env_var_name: env_var_name
+      properties:
+        mlp_secret_name:
+          pattern: ^[-._a-zA-Z0-9]+$
+          type: string
+        env_var_name:
+          pattern: ^[a-zA-Z0-9_]*$
+          type: string
+      required:
+      - env_var_name
+      - mlp_secret_name
+      type: object
     EnvVar:
       example:
         name: name

diff --git a/api/api/specs/common.yaml b/api/api/specs/common.yaml
@@ -35,6 +35,19 @@ components:
         value:
           type: "string"
 
+    MountedMLPSecret:
+      type: "object"
+      required:
+        - mlp_secret_name
+        - env_var_name
+      properties:
+        mlp_secret_name:
+          type: "string"
+          pattern: '^[-._a-zA-Z0-9]+$'
+        env_var_name:
+          type: "string"
+          pattern: '^[a-zA-Z0-9_]*$'
+
     pagination.Paging:
       type: "object"
       properties:

diff --git a/api/api/specs/jobs.yaml b/api/api/specs/jobs.yaml
@@ -417,6 +417,10 @@ components:
         service_account_name:
           type: string
           x-go-custom-tag: validate:"required"
+        secrets:
+          type: array
+          items:
+            $ref: "common.yaml#/components/schemas/MountedMLPSecret"
         resources:
           $ref: "#/components/schemas/EnsemblingResources"
         run_id:

diff --git a/api/api/specs/routers.yaml b/api/api/specs/routers.yaml
@@ -782,6 +782,7 @@ components:
         - timeout
         - port
         - env
+        - secrets
       properties:
         id:
           $ref: "common.yaml#/components/schemas/Id"
@@ -801,6 +802,10 @@ components:
           type: "array"
           items:
             $ref: "common.yaml#/components/schemas/EnvVar"
+        secrets:
+          type: "array"
+          items:
+            $ref: "common.yaml#/components/schemas/MountedMLPSecret"
         service_account:
           type: "string"
           description: >
@@ -888,6 +893,7 @@ components:
         - timeout
         - port
         - env
+        - secrets
       properties:
         image:
           type: "string"
@@ -906,6 +912,10 @@ components:
           type: "array"
           items:
             $ref: "common.yaml#/components/schemas/EnvVar"
+        secrets:
+          type: "array"
+          items:
+            $ref: "common.yaml#/components/schemas/MountedMLPSecret"
         service_account:
           type: "string"
           description: >
@@ -923,6 +933,8 @@ components:
         - ensembler_id
         - resource_request
         - timeout
+        - env
+        - secrets
       properties:
         project_id:
           type: "integer"
@@ -938,6 +950,10 @@ components:
           type: "array"
           items:
             $ref: "common.yaml#/components/schemas/EnvVar"
+        secrets:
+          type: "array"
+          items:
+            $ref: "common.yaml#/components/schemas/MountedMLPSecret"
 
     ResourceRequest:
       type: "object"

diff --git a/api/db-migrations/000016_add_secrets_columns.down.sql b/api/db-migrations/000016_add_secrets_columns.down.sql
@@ -0,0 +1,7 @@
+-- Remove secrets column for enrichers
+ALTER TABLE enrichers DROP COLUMN secrets;
+
+-- Remove secrets field in docker_config and pyfunc_config columns for ensemblers
+UPDATE ensembler_configs set docker_config = docker_config - 'secrets' WHERE docker_config IS NOT NULL;
+
+UPDATE ensembler_configs set pyfunc_config = pyfunc_config - 'secrets' WHERE pyfunc_config IS NOT NULL;
diff --git a/api/db-migrations/000016_add_secrets_columns.up.sql b/api/db-migrations/000016_add_secrets_columns.up.sql
@@ -0,0 +1,7 @@
+-- Create secrets column for enrichers
+ALTER TABLE enrichers ADD COLUMN secrets jsonb NOT NULL DEFAULT '[]'::jsonb;
+
+-- Create secrets field in docker_config and pyfunc_config columns for ensemblers
+UPDATE ensembler_configs SET docker_config = jsonb_set(docker_config, '{secrets}', '[]'::jsonb) WHERE docker_config IS NOT NULL AND docker_config->'secrets' IS NULL;
+
+UPDATE ensembler_configs SET pyfunc_config = jsonb_set(pyfunc_config, '{secrets}', '[]'::jsonb) WHERE pyfunc_config IS NOT NULL AND pyfunc_config->'secrets' IS NULL;