Define service resources from within services.{service}.resources val…

…ues (#474)

This deprecates the use of resources.memory.{service}, except for job resources (e.g. app-init/app-migrate, which don't have a similar abstraction yet)

Also the helm/_twig/values.yaml/resources.yml.twig is also deprecated, as unnecessary with serialisation of resources to yaml

src/_base/harness/attributes/common.yml

@@ -348,19 +348,6 @@ attributes.default:
     memory:
       app_init: "1024Mi"
       app_migrate: "1024Mi"
-      console: "2048Mi"
-      cron: "1024Mi"
-      elasticsearch: "1024Mi"
-      mysql: "512Mi"
-      nginx: "100Mi"
-      php_fpm: "1024Mi"
-      php_fpm_exporter: "100Mi"
-      postgres: "512Mi"
-      rabbitmq: "1024Mi"
-      redis: "256Mi"
-      redis_session: "1024Mi"
-      tideways: "256Mi"
-      varnish: "1124Mi"
 
   replicas:
     varnish: 1

src/_base/harness/attributes/docker-base.yml

@@ -39,6 +39,8 @@ attributes:
       publish: true
       environment:
         FPM_HOST: php-fpm
+      resources:
+        memory: "100Mi"
     console:
       enabled: true
       image: = @('docker.repository') ~ ':' ~ @('app.version') ~ '-console'
@@ -49,19 +51,27 @@ attributes:
         HAS_VARNISH: "= @('services.varnish.enabled') ? 'true' : 'false'"
       environment_secrets:
         DB_ADMIN_PASS: = @('database.root_pass')
+      resources:
+        memory: "2048Mi"
     cron:
       enabled: "= 'cron' in @('app.services')"
       image: = @('docker.repository') ~ ':' ~ @('app.version') ~ '-cron'
       publish: "= @('services.cron.enabled')"
+      resources:
+        memory: "1024Mi"
     php-fpm:
       image: = @('docker.repository') ~ ':' ~ @('app.version') ~ '-php-fpm'
       publish: true
       environment:
         AUTOSTART_PHP_FPM: "true"
+      resources:
+        memory: "1024Mi"
     php-fpm-exporter:
       image: hipages/php-fpm_exporter
       environment:
         PHP_FPM_SCRAPE_URI: = php_fpm_exporter_scrape_url('php-fpm', @('php-fpm.pools'))
+      resources:
+        memory: "100Mi"
     blackfire:
       enabled: "= 'blackfire' in @('app.services')"
       image: blackfire/blackfire:latest
@@ -76,6 +86,8 @@ attributes:
     elasticsearch:
       enabled: "= 'elasticsearch' in @('app.services')"
       image: = @('elasticsearch.image') ~ ':' ~ @('elasticsearch.tag')
+      resources:
+        memory: "1024Mi"
     memcached:
       enabled: "= 'memcached' in @('app.services')"
       image: memcached:1-alpine
@@ -89,6 +101,8 @@ attributes:
       environment_secrets:
         MYSQL_PASSWORD: = @('database.pass')
         MYSQL_ROOT_PASSWORD: = @('database.root_pass')
+      resources:
+        memory: "512Mi"
     postgres:
       enabled: "= 'postgres' in @('app.services')"
       image: postgres:9.6
@@ -98,6 +112,8 @@ attributes:
         PGDATA: /var/lib/postgresql/data/pgdata
       environment_secrets:
         POSTGRES_PASSWORD: = @('database.pass')
+      resources:
+        memory: "512Mi"
     rabbitmq:
       enabled: "= 'rabbitmq' in @('app.services')"
       image: = @('rabbitmq.image') ~ ':' ~ @('rabbitmq.tag')
@@ -107,12 +123,18 @@ attributes:
       environment_secrets:
         RABBITMQ_DEFAULT_PASS: = @('rabbitmq.password')
         RABBITMQ_ERLANG_COOKIE: = @('rabbitmq.erlang_cookie')
+      resources:
+        memory: "1024Mi"
     redis:
       enabled: "= 'redis' in @('app.services')"
       image: redis:5-alpine
+      resources:
+        memory: "256Mi"
     redis_session:
       enabled: "= 'redis_session' in @('app.services')"
       image: redis:5-alpine
+      resources:
+        memory: "1024Mi"
     relay:
       enabled: true
       publish: false
@@ -132,11 +154,15 @@ attributes:
       environment:
         TIDEWAYS_HOSTNAME: = @('hostname')
         TIDEWAYS_ENV: production
+      resources:
+        memory: "256Mi"
     varnish:
       enabled: "= 'varnish' in @('app.services')"
       image: varnish:6
       environment:
         VARNISH_SIZE: "1024M"
+      resources:
+        memory: "1124Mi"
     # webapp is by default two services combined, nginx and php-fpm
     webapp:
       enabled: true
@@ -219,22 +245,42 @@ attributes:
         memory:
           app_init: "1024Mi"
           app_migrate: "1024Mi"
-          console: "1024Mi"
-          cron: "1024Mi"
-          elasticsearch: "1024Mi"
-          mysql: "512Mi"
-          nginx: "64Mi"
-          php_fpm: "1024Mi"
-          php_fpm_exporter: "32Mi"
-          postgres: "512Mi"
-          redis: "64Mi"
-          redis_session: "64Mi"
-          tideways: "256Mi"
-          varnish: "1124Mi"
       services:
         console:
           enabled: false
+          resources:
+            memory: "1024Mi"
+        cron:
+          resources:
+            memory: "1024Mi"
+        elasticsearch:
+          resources:
+            memory: "1024Mi"
+        mysql:
+          resources:
+            memory: "512Mi"
+        nginx:
+          resources:
+            memory: "64Mi"
         php-base:
           environment:
             SMTP_HOST: = @('pipeline.preview.smtp.host')
             SMTP_PORT: = @('pipeline.preview.smtp.port')
+        php-fpm:
+          resources:
+            memory: "1024Mi"
+        php-fpm-exporter:
+          resources:
+            memory: "32Mi"
+        postgres:
+          resources:
+            memory: "512Mi"
+        redis:
+          resources:
+            memory: "64Mi"
+        redis_session:
+          resources:
+            memory: "64Mi"
+        tideways:
+          resources:
+            memory: "256Mi"

src/_base/harness/config/functions.yml

@@ -88,6 +88,7 @@ function('filter_local_services', [services]): |
         case 'enabled':
         case 'environment':
         case 'image':
+        case 'resources':
           $filteredService[$key] = $value;
       }
     }

src/_base/helm/app/_twig/values.yaml/resources.yml.twig

@@ -1,19 +1,3 @@
-  cpu:
-    requests:
-    limits:
-  memory:
-    app_init: {{ @('resources.memory.app_init') }}
-    app_migrate: {{ @('resources.memory.app_migrate') }}
-    console: {{ @('resources.memory.console') }}
-    cron: {{ @('resources.memory.cron') }}
-    elasticsearch: {{ @('resources.memory.elasticsearch') }}
-    mysql: {{ @('resources.memory.mysql') }}
-    nginx: {{ @('resources.memory.nginx') }}
-    php_fpm: {{ @('resources.memory.php_fpm') }}
-    php_fpm_exporter: {{ @('resources.memory.php_fpm_exporter') }}
-    postgres: {{ @('resources.memory.postgres') }}
-    rabbitmq: {{ @('resources.memory.rabbitmq') }}
-    redis: {{ @('resources.memory.redis') }}
-    redis_session: {{ @('resources.memory.redis_session') }}
-    tideways: "{{ @('resources.memory.tideways') }}"
-    varnish: {{ @('resources.memory.varnish') }}
+{# this file is deprecated, and instead resources attribute is serialised into the values directly #}
+{# in the case of services, resource attribute is further deprecated and available continuing forward in services.*.resources #}
+{{ to_nice_yaml(@('resources'), 2, 2) | raw }}

src/_base/helm/app/templates/application/app-init.yaml

@@ -1,8 +1,8 @@
-{{- $service := mergeOverwrite (dict) (index .Values.docker.services "php-base") (index .Values.docker.services "console") (dict "environment" .Values.environment) -}}
+{{- with mergeOverwrite (dict) (index .Values.docker.services "php-base") (index .Values.docker.services "console") (dict "environment" .Values.environment) -}}
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: {{ .Values.resourcePrefix }}app-init
+  name: {{ $.Values.resourcePrefix }}app-init
   annotations:
     helm.sh/hook: "post-install"
     helm.sh/hook-delete-policy: "before-hook-creation"
@@ -16,7 +16,7 @@ spec:
       restartPolicy: Never
       containers:
       - env:
-        {{- range $key, $value := $service.environment }}
+        {{- range $key, $value := .environment }}
         {{- $tp := typeOf $value }}
         - name: {{ $key | quote }}
         {{- if eq $tp "string" }}
@@ -25,15 +25,15 @@ spec:
           value: {{ $value | quote }}
         {{- end }}
         {{- end }}
-        {{- if $service.environment_secrets }}
+        {{- if .environment_secrets }}
         envFrom:
           - secretRef:
-              name: {{ .Values.resourcePrefix }}console
+              name: {{ $.Values.resourcePrefix }}console
         {{- end }}
-        image: {{ .Values.docker.image.console | default $service.image | quote }}
+        image: {{ $.Values.docker.image.console | default .image | quote }}
         imagePullPolicy: Always
         name: app-init
-{{ if eq .Values.ingress.type "istio" }}
+{{ if eq $.Values.ingress.type "istio" }}
         command: ["/bin/sh"]
         args: ["-c", "/entrypoint.sh app init ; code=$? ; curl -vv -XPOST http://127.0.0.1:15020/quitquitquit ; exit $code"]
 {{ else }}
@@ -42,22 +42,23 @@ spec:
 {{ end }}
         resources:
           limits:
-            memory: {{ .Values.resources.memory.app_init }}
+            memory: {{ $.Values.resources.memory.app_init }}
           requests:
-            memory: {{ .Values.resources.memory.app_init }}
+            memory: {{ $.Values.resources.memory.app_init }}
         volumeMounts:
-        {{- if not (eq "" (include "application.volumeMounts.console" .)) }}
-        {{- include "application.volumeMounts.console" . | indent 8 }}
+        {{- if not (eq "" (include "application.volumeMounts.console" $)) }}
+        {{- include "application.volumeMounts.console" $ | indent 8 }}
         {{- else }} []
         {{- end }}
-{{- if .Values.docker.image_pull_config }}
+{{- if $.Values.docker.image_pull_config }}
       imagePullSecrets:
-      - name: {{ .Values.resourcePrefix }}image-pull-config
+      - name: {{ $.Values.resourcePrefix }}image-pull-config
 {{- end }}
       restartPolicy: Never
       enableServiceLinks: false
       volumes:
-      {{- if not (eq "" (include "application.volumes.console" .)) }}
-      {{- include "application.volumes.console" . | indent 6 }}
+      {{- if not (eq "" (include "application.volumes.console" $)) }}
+      {{- include "application.volumes.console" $ | indent 6 }}
       {{- else }} []
       {{- end }}
+{{- end }}

src/_base/helm/app/templates/application/app-migrate.yaml

@@ -1,8 +1,8 @@
-{{- $service := mergeOverwrite (dict) (index .Values.docker.services "php-base") (index .Values.docker.services "console") (dict "environment" .Values.environment) -}}
+{{- with mergeOverwrite (dict) (index .Values.docker.services "php-base") (index .Values.docker.services "console") (dict "environment" .Values.environment) -}}
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: {{ .Values.resourcePrefix }}app-migrate
+  name: {{ $.Values.resourcePrefix }}app-migrate
   annotations:
     helm.sh/hook: "pre-upgrade"
     helm.sh/hook-delete-policy: "before-hook-creation"
@@ -14,7 +14,7 @@ spec:
     spec:
       containers:
       - env:
-        {{- range $key, $value := $service.environment }}
+        {{- range $key, $value := .environment }}
         {{- $tp := typeOf $value }}
         - name: {{ $key | quote }}
         {{- if eq $tp "string" }}
@@ -23,15 +23,15 @@ spec:
           value: {{ $value | quote }}
         {{- end }}
         {{- end }}
-        {{- if $service.environment_secrets }}
+        {{- if .environment_secrets }}
         envFrom:
           - secretRef:
-              name: {{ .Values.resourcePrefix }}console
+              name: {{ $.Values.resourcePrefix }}console
         {{- end }}
-        image: {{ .Values.docker.image.console | default $service.image }}
+        image: {{ $.Values.docker.image.console | default .image }}
         imagePullPolicy: Always
         name: app-migrate
-{{ if eq .Values.ingress.type "istio" }}
+{{ if eq $.Values.ingress.type "istio" }}
         command: ["/bin/sh"]
         args: ["-c", "/entrypoint.sh app migrate ; code=$? ; curl -vv -XPOST http://127.0.0.1:15020/quitquitquit ; exit $code"]
 {{ else }}
@@ -40,22 +40,23 @@ spec:
 {{ end }}
         resources:
           limits:
-            memory: {{ .Values.resources.memory.app_migrate }}
+            memory: {{ $.Values.resources.memory.app_migrate }}
           requests:
-            memory: {{ .Values.resources.memory.app_migrate }}
+            memory: {{ $.Values.resources.memory.app_migrate }}
         volumeMounts:
-        {{- if not (eq "" (include "application.volumeMounts.console" .)) }}
-        {{- include "application.volumeMounts.console" . | indent 8 }}
+        {{- if not (eq "" (include "application.volumeMounts.console" $)) }}
+        {{- include "application.volumeMounts.console" $ | indent 8 }}
         {{- else }} []
         {{- end }}
-{{- if .Values.docker.image_pull_config }}
+{{- if $.Values.docker.image_pull_config }}
       imagePullSecrets:
-      - name: {{ .Values.resourcePrefix }}image-pull-config
+      - name: {{ $.Values.resourcePrefix }}image-pull-config
 {{- end }}
       restartPolicy: Never
       enableServiceLinks: false
       volumes:
-      {{- if not (eq "" (include "application.volumes.console" .)) }}
-      {{- include "application.volumes.console" . | indent 6 }}
+      {{- if not (eq "" (include "application.volumes.console" $)) }}
+      {{- include "application.volumes.console" $ | indent 6 }}
       {{- else }} []
       {{- end }}
+{{- end }}