Merge pull request #670 from vmware-tanzu/data-values-dir

Support loading Data Values files in the same ways "ordinary" files are loaded

examples/data-values-directory/README.md

@@ -0,0 +1,23 @@
+This example demonstrates how ytt handles specifying a directory for Data Values
+inputs. (originally https://kubernetes.slack.com/archives/CH8KCCKA5/p1651167583289939)
+
+With this:
+```
+├── config
+│   ├── config.yml
+│   └── values-schema.yml
+└── values                   <-- any YAML under here is assumed to be plain Data Values
+    ├── appdev-overrides     <-- sorted by full pathname, alphabetically
+    │   └── values.yaml
+    └── operator-overrides
+        ├── 50-operations-overrides.yml
+        ├── 99-opssec-overrides.yml
+        └── approvals.toml   <-- non YAML files are ignored.
+```
+
+Executing this:
+
+```console
+$ ytt -f examples/data-values-directory/config.yml \
+      --data-values-file examples/data-values-directory/values/
+```

examples/data-values-directory/config/config.yml

@@ -0,0 +1,8 @@
+#! In the real-world, this configuration would be the manifests that describe
+#! this application's deployment. For this example, we simply render the
+#! net values; this is the point of this example.
+
+#@ load("@ytt:data", "data")
+
+---
+values: #@ data.values

examples/data-values-directory/config/values-schema.yml

@@ -0,0 +1,10 @@
+#@data/values-schema
+---
+name: suggestion-service
+instances: 0
+accept_insecure_conns: true
+#@schema/nullable
+cache:
+  driver: ""
+  #@schema/type any=True
+  config: {}

examples/data-values-directory/expected.txt

@@ -0,0 +1,12 @@
+values:
+  name: suggestion-service
+  instances: 4
+  accept_insecure_conns: false
+  cache:
+    driver: redis
+    config:
+      maxEntries: 1024
+      strategy: MRU
+      host: localhost:6379
+      tls-client-cert-file: client.crt
+      tls-client-key-file: client.key

examples/data-values-directory/run.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -e
+
+./ytt -f examples/data-values-directory/config/ \
+      --data-values-file examples/data-values-directory/values/

examples/data-values-directory/values/appdev-overrides/values.yaml

@@ -0,0 +1,8 @@
+---
+instances: 1
+accept_insecure_conns: true
+cache:
+  driver: in-memory
+  config:
+    maxEntries: 1024
+    strategy: MRU

examples/data-values-directory/values/operator-overrides/50-operations-overrides.yml

@@ -0,0 +1,10 @@
+---
+# 2022-05-13: avg + rms over last 3 months
+instances: 4
+
+cache:
+  driver: redis
+  config:
+    host: localhost:6379
+    tls-client-cert-file: client.crt
+    tls-client-key-file: client.key

examples/data-values-directory/values/operator-overrides/99-opssec-overrides.yml

@@ -0,0 +1,3 @@
+---
+# Policy 1.31a: all north-south connections must be encrypted.
+accept_insecure_conns: false

examples/data-values-directory/values/operator-overrides/approvals.toml

@@ -0,0 +1,7 @@
+[[approvals]]
+    dept = "operations"
+    user = "[email protected]"
+
+[[approvals]]
+    dept = "Operations Security"
+    user = "[email protected]"

examples/overlay-files/expected.txt

@@ -0,0 +1,251 @@
+name: bosh
+releases:
+- name: bosh
+  version: 268.6.0
+  url: https://s3.amazonaws.com/...
+  sha1: 480b15380f446bcd6fb86511e1ad39b4f1019e37
+- name: bpm
+  version: 0.12.3
+  url: https://s3.amazonaws.com/...
+  sha1: 54fbf8e2ecf14c69ee761ddde0624edd228ac478
+- name: os-conf
+  version: 18
+  url: https://bosh.io/d/github.com/cloudfoundry/os-conf-release?v=18
+  sha1: 78d79f08ff5001cc2a24f572837c7a9c59a0e796
+resource_pools:
+- name: vms
+  network: default
+  env:
+    bosh:
+      password: '*'
+      mbus:
+        cert: ((mbus_bootstrap_ssl))
+disk_pools:
+- name: disks
+  disk_size: 65536
+networks:
+- name: default
+  type: manual
+  subnets:
+  - range: ((internal_cidr))
+    gateway: ((internal_gw))
+    static:
+    - ((internal_ip))
+    dns:
+    - 8.8.8.8
+instance_groups:
+- name: bosh
+  instances: 1
+  jobs:
+  - name: bpm
+    release: bpm
+  - name: nats
+    release: bosh
+  - name: postgres-10
+    release: bosh
+  - name: blobstore
+    release: bosh
+  - name: director
+    release: bosh
+  - name: health_monitor
+    release: bosh
+  - name: user_add
+    release: os-conf
+    properties:
+      users:
+      - name: jumpbox
+        public_key: ((jumpbox_ssh.public_key))
+  resource_pool: vms
+  persistent_disk_pool: disks
+  networks:
+  - name: default
+    static_ips:
+    - ((internal_ip))
+  properties:
+    agent:
+      mbus: nats://nats:((nats_password))@((internal_ip)):4222
+      env:
+        bosh:
+          blobstores:
+          - provider: dav
+            options:
+              endpoint: https://((internal_ip)):25250
+              user: agent
+              password: ((blobstore_agent_password))
+              tls:
+                cert:
+                  ca: ((blobstore_ca.certificate))
+    nats:
+      address: ((internal_ip))
+      user: nats
+      password: ((nats_password))
+      tls:
+        ca: ((nats_server_tls.ca))
+        client_ca:
+          certificate: ((nats_ca.certificate))
+          private_key: ((nats_ca.private_key))
+        server:
+          certificate: ((nats_server_tls.certificate))
+          private_key: ((nats_server_tls.private_key))
+        director:
+          certificate: ((nats_clients_director_tls.certificate))
+          private_key: ((nats_clients_director_tls.private_key))
+        health_monitor:
+          certificate: ((nats_clients_health_monitor_tls.certificate))
+          private_key: ((nats_clients_health_monitor_tls.private_key))
+    postgres:
+      listen_address: 127.0.0.1
+      host: 127.0.0.1
+      user: postgres
+      password: ((postgres_password))
+      database: bosh
+      adapter: postgres
+    blobstore:
+      address: ((internal_ip))
+      port: 25250
+      provider: dav
+      director:
+        user: director
+        password: ((blobstore_director_password))
+      agent:
+        user: agent
+        password: ((blobstore_agent_password))
+      tls:
+        cert:
+          ca: ((blobstore_ca.certificate))
+          certificate: ((blobstore_server_tls.certificate))
+          private_key: ((blobstore_server_tls.private_key))
+    director:
+      address: 127.0.0.1
+      name: ((director_name))
+      db:
+        listen_address: 127.0.0.1
+        host: 127.0.0.1
+        user: postgres
+        password: ((postgres_password))
+        database: bosh
+        adapter: postgres
+      flush_arp: true
+      enable_post_deploy: true
+      generate_vm_passwords: true
+      enable_dedicated_status_worker: true
+      enable_nats_delivered_templates: true
+      workers: 4
+      local_dns:
+        enabled: true
+      events:
+        record_events: true
+      ssl:
+        key: ((director_ssl.private_key))
+        cert: ((director_ssl.certificate))
+      user_management:
+        provider: local
+        local:
+          users:
+          - name: admin
+            password: ((admin_password))
+          - name: hm
+            password: ((hm_password))
+      default_ssh_options:
+        gateway_user: jumpbox
+    hm:
+      director_account:
+        user: hm
+        password: ((hm_password))
+        ca_cert: ((director_ssl.ca))
+      resurrector_enabled: true
+    ntp:
+    - time1.google.com
+    - time2.google.com
+    - time3.google.com
+    - time4.google.com
+cloud_provider:
+  mbus: https://mbus:((mbus_bootstrap_password))@((internal_ip)):6868
+  cert: ((mbus_bootstrap_ssl))
+  properties:
+    agent:
+      mbus: https://mbus:((mbus_bootstrap_password))@0.0.0.0:6868
+    blobstore:
+      provider: local
+      path: /var/vcap/micro_bosh/data/cache
+    ntp:
+    - time1.google.com
+    - time2.google.com
+    - time3.google.com
+    - time4.google.com
+variables:
+- name: admin_password
+  type: password
+- name: blobstore_director_password
+  type: password
+- name: blobstore_agent_password
+  type: password
+- name: hm_password
+  type: password
+- name: mbus_bootstrap_password
+  type: password
+- name: nats_password
+  type: password
+- name: postgres_password
+  type: password
+- name: default_ca
+  type: certificate
+  options:
+    is_ca: true
+    common_name: ca
+- name: mbus_bootstrap_ssl
+  type: certificate
+  options:
+    ca: default_ca
+    common_name: ((internal_ip))
+    alternative_names:
+    - ((internal_ip))
+- name: director_ssl
+  type: certificate
+  options:
+    ca: default_ca
+    common_name: ((internal_ip))
+    alternative_names:
+    - ((internal_ip))
+- name: nats_ca
+  type: certificate
+  options:
+    is_ca: true
+    common_name: default.nats-ca.bosh-internal
+- name: nats_server_tls
+  type: certificate
+  options:
+    ca: nats_ca
+    common_name: default.nats.bosh-internal
+    alternative_names:
+    - ((internal_ip))
+    extended_key_usage:
+    - server_auth
+- name: nats_clients_director_tls
+  type: certificate
+  options:
+    ca: nats_ca
+    common_name: default.director.bosh-internal
+    extended_key_usage:
+    - client_auth
+- name: nats_clients_health_monitor_tls
+  type: certificate
+  options:
+    ca: nats_ca
+    common_name: default.hm.bosh-internal
+    extended_key_usage:
+    - client_auth
+- name: blobstore_ca
+  type: certificate
+  options:
+    is_ca: true
+    common_name: default.blobstore-ca.bosh-internal
+- name: blobstore_server_tls
+  type: certificate
+  options:
+    ca: blobstore_ca
+    common_name: ((internal_ip))
+    alternative_names:
+    - ((internal_ip))
+- name: jumpbox_ssh
+  type: ssh