From 4dd8f6ef97eeb1a858a83cf7f281772bafd579aa Mon Sep 17 00:00:00 2001 From: Muhammad Saud Khan Date: Thu, 4 Jan 2024 18:37:58 +0100 Subject: [PATCH] chore: refactor script and update getting-started.md --- deployment/local/scripts/init-values.sh | 259 ++++++++++++++++++++++++ docs/GETTING-STARTED.md | 13 +- 2 files changed, 268 insertions(+), 4 deletions(-) create mode 100644 deployment/local/scripts/init-values.sh diff --git a/deployment/local/scripts/init-values.sh b/deployment/local/scripts/init-values.sh new file mode 100644 index 000000000..ea66d911e --- /dev/null +++ b/deployment/local/scripts/init-values.sh @@ -0,0 +1,259 @@ +#!/bin/bash +################################################################################# +# Catena-X - Digital Product Pass Application +# +# Copyright (c) 2022, 2024 BASF SE, BMW AG, Henkel AG & Co. KGaA +# Copyright (c) 2022, 2024 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied. See the +# License for the specific language govern in permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 +################################################################################## + + +set -o errexit +set -o errtrace +set -o pipefail +set -o nounset + +SET_VALUES=$1 +GITHUB_TOKEN=$2 +VAULT_ADDRESS=$3 +DPP_VALUES_FILE="./../../../charts/digital-product-pass/values-dev.yaml" +EDC_CONSUMER_VALUES_FILE="./../../infrastructure/edc-consumer/values.yaml" +EDC_PROVIDER_VALUES_FILE="./../../infrastructure/edc-provider/values.yaml" +REGISTRY_VALUES_FILE="./../../infrastructure/registry/values.yaml" + +export GH_TOKEN=${GITHUB_TOKEN} +export VAULT_ADDR=${VAULT_ADDRESS} + +VALUE="" + +get_value() { + + field_name=$1 + path=$2 + + VALUE=$(vault kv get -field="${field_name}" "${path}") + +} + +echo "Login into Hashicorp vault..." +vault login -method=github token=$GH_TOKEN +echo +echo "Retrieve secrets from Hashicorp vault..." +echo + +get_value "api.key" "material-pass/dev/edc/oauth"; EDC_API_AUTH_KEY=$VALUE +get_value "client.id" "material-pass/dev/edc/oauth"; EDC_OAUTH_CLIENT_ID=$VALUE +get_value "realm" "material-pass/dev/oauth"; REALM=$VALUE +get_value "appId" "material-pass/dev/oauth"; APP_ID=$VALUE +get_value "clientId" "material-pass/dev/backend"; DPP_CLIENT_ID=$VALUE +get_value "clientSecret" "material-pass/dev/backend"; DPP_CLIENT_SECRET=$VALUE +get_value "miwUrl" "material-pass/dev/edc/ssi"; MIW_URL=$VALUE +get_value "authorityId" "material-pass/dev/edc/ssi"; AUTHORITY_ID=$VALUE +get_value "clientId" "material-pass/dev/edc/ssi"; SSI_CLIENT_ID=$VALUE +get_value "user" "material-pass/dev/edc/database"; POSTGRES_DB_USER=$VALUE +get_value "password" "material-pass/dev/edc/database"; POSTGRES_DB_PASSWORD=$VALUE +get_value "bpnNumber" "material-pass/dev/edc/participant"; PARTICIPANT_ID=$VALUE +get_value "vault.hashicorp.url" "material-pass/dev/edc/vault"; VAULT_HASHICORP_URL=$VALUE +get_value "vault.hashicorp.api.secret.path" "material-pass/dev/edc/vault"; VAULT_HASHICORP_SECRET_PATH=$VALUE +get_value "vault.hashicorp.token" "material-pass/dev/edc/vault"; VAULT_HASHICORP_TOKEN=$VALUE +get_value "user" "material-pass/dev/irs/minio"; IRS_MINIO_USER=$VALUE +get_value "password" "material-pass/dev/irs/minio"; IRS_MINIO_PASSWORD=$VALUE +get_value "clientId" "material-pass/dev/irs/keycloak/oauth2"; IRS_KEYCLOAK_CLIENT_ID=$VALUE +get_value "clientSecret" "material-pass/dev/irs/keycloak/oauth2"; IRS_KEYCLOAK_CLIENT_SECRET=$VALUE +get_value "database.user" "material-pass/dev/aasregistry"; REGISTRY_USER=$VALUE +get_value "database.password" "material-pass/dev/aasregistry"; REGISTRY_PASSWORD=$VALUE + + +set_values () { + + ### DIGITAL PRODUCT PASS APP + + echo "Replace client credentials with actual values in digital-product-pass values file..." + echo + sed -i "s||${EDC_API_AUTH_KEY}|g" ${DPP_VALUES_FILE}; + sed -i "s||${PARTICIPANT_ID}|g" ${DPP_VALUES_FILE}; + sed -i "s||${REALM}|g" ${DPP_VALUES_FILE}; + sed -i "s||${APP_ID}|g" ${DPP_VALUES_FILE}; + sed -i "s||${DPP_CLIENT_ID}|g" ${DPP_VALUES_FILE}; + sed -i "s||${DPP_CLIENT_SECRET}|g" ${DPP_VALUES_FILE}; + + ### CONSUMER EDC CONNECTOR + + echo "Replace secrets with actual values in edc-consumer values file..." + echo + + sed -i "s||${PARTICIPANT_ID}|g" ${EDC_CONSUMER_VALUES_FILE}; + + # Postgres DB + sed -i "s||${POSTGRES_DB_USER}|g" ${EDC_CONSUMER_VALUES_FILE}; + sed -i "s||${POSTGRES_DB_PASSWORD}|g" ${EDC_CONSUMER_VALUES_FILE}; + + # Hashicorp vault + sed -i "s||${VAULT_HASHICORP_URL}|g" ${EDC_CONSUMER_VALUES_FILE}; + sed -i "s||${VAULT_HASHICORP_SECRET_PATH}|g" ${EDC_CONSUMER_VALUES_FILE}; + sed -i "s||${VAULT_HASHICORP_TOKEN}|g" ${EDC_CONSUMER_VALUES_FILE}; + + # EDC Oauth + sed -i "s|authKey: |authKey: ${EDC_API_AUTH_KEY}|g" ${EDC_CONSUMER_VALUES_FILE}; + sed -i "s|secret: |secret: ${EDC_API_AUTH_KEY}|g" ${EDC_CONSUMER_VALUES_FILE}; + + # ssi + sed -i "s||${MIW_URL}|g" ${EDC_CONSUMER_VALUES_FILE}; + sed -i "s||${AUTHORITY_ID}|g" ${EDC_CONSUMER_VALUES_FILE}; + sed -i "s||${SSI_CLIENT_ID}|g" ${EDC_CONSUMER_VALUES_FILE}; + + # irs + sed -i "s||${IRS_MINIO_USER}|g" ${EDC_CONSUMER_VALUES_FILE}; + sed -i "s||${IRS_MINIO_PASSWORD}|g" ${EDC_CONSUMER_VALUES_FILE}; + sed -i "s||${IRS_KEYCLOAK_CLIENT_ID}|g" ${EDC_CONSUMER_VALUES_FILE}; + sed -i "s||${IRS_KEYCLOAK_CLIENT_SECRET}|g" ${EDC_CONSUMER_VALUES_FILE}; + + + ### PROVIDER EDC CONNECTOR + + echo "Replace secrets with actual values in edc-provider values file..." + echo + + sed -i "s||${PARTICIPANT_ID}|g" ${EDC_PROVIDER_VALUES_FILE}; + + # Postgres DB + sed -i "s||${POSTGRES_DB_USER}|g" ${EDC_PROVIDER_VALUES_FILE}; + sed -i "s||${POSTGRES_DB_PASSWORD}|g" ${EDC_PROVIDER_VALUES_FILE}; + + # Hashicorp vault + sed -i "s||${VAULT_HASHICORP_URL}|g" ${EDC_PROVIDER_VALUES_FILE}; + sed -i "s||${VAULT_HASHICORP_SECRET_PATH}|g" ${EDC_PROVIDER_VALUES_FILE}; + sed -i "s||${VAULT_HASHICORP_TOKEN}|g" ${EDC_PROVIDER_VALUES_FILE}; + + # EDC Oauth + sed -i "s|authKey: |authKey: ${EDC_API_AUTH_KEY}|g" ${EDC_PROVIDER_VALUES_FILE}; + sed -i "s|secret: |secret: ${EDC_API_AUTH_KEY}|g" ${EDC_PROVIDER_VALUES_FILE}; + + + # ssi + sed -i "s||${MIW_URL}|g" ${EDC_PROVIDER_VALUES_FILE}; + sed -i "s||${AUTHORITY_ID}|g" ${EDC_PROVIDER_VALUES_FILE}; + sed -i "s||${SSI_CLIENT_ID}|g" ${EDC_PROVIDER_VALUES_FILE}; + + ### PROVIDER REGISTRY + + echo "Replace secrets with actual values in registry values file..." + echo + + # Postgres DB + sed -i "s||${REGISTRY_USER}|g" ${REGISTRY_VALUES_FILE}; + sed -i "s||${REGISTRY_PASSWORD}|g" ${REGISTRY_VALUES_FILE}; +} + +unset_values() { + + ### DIGITAL PRODUCT PASS APP + + echo "Replace actual values with placholders in digital-product-pass values file..." + echo + sed -i "s|${EDC_API_AUTH_KEY}||g" ${DPP_VALUES_FILE}; + sed -i "s|${PARTICIPANT_ID}||g" ${DPP_VALUES_FILE}; + sed -i "s|${REALM}||g" ${DPP_VALUES_FILE}; + sed -i "s|${APP_ID}||g" ${DPP_VALUES_FILE}; + sed -i "s|${DPP_CLIENT_ID}||g" ${DPP_VALUES_FILE}; + sed -i "s|${DPP_CLIENT_SECRET}||g" ${DPP_VALUES_FILE}; + + + ### CONSUMER EDC CONNECTOR + + echo "Replace actual values with placholders in edc-consumer values file..." + echo + + sed -i "s|${PARTICIPANT_ID}||g" ${EDC_CONSUMER_VALUES_FILE}; + + # Postgres DB + sed -i "s|${POSTGRES_DB_USER}||g" ${EDC_CONSUMER_VALUES_FILE}; + sed -i "s|password: ${POSTGRES_DB_PASSWORD}|password: |g" ${EDC_CONSUMER_VALUES_FILE}; + + # Hashicorp vault + sed -i "s|${VAULT_HASHICORP_URL}||g" ${EDC_CONSUMER_VALUES_FILE}; + sed -i "s|${VAULT_HASHICORP_SECRET_PATH}||g" ${EDC_CONSUMER_VALUES_FILE}; + sed -i "s|${VAULT_HASHICORP_TOKEN}||g" ${EDC_CONSUMER_VALUES_FILE}; + + # EDC Oauth + sed -i "s|authKey: ${EDC_API_AUTH_KEY}|authKey: |g" ${EDC_CONSUMER_VALUES_FILE}; + sed -i "s|secret: ${EDC_API_AUTH_KEY}|secret: |g" ${EDC_CONSUMER_VALUES_FILE}; + + # irs + sed -i "s|${IRS_MINIO_USER}||g" ${EDC_CONSUMER_VALUES_FILE}; + sed -i "s|${IRS_MINIO_PASSWORD}||g" ${EDC_CONSUMER_VALUES_FILE}; + sed -i "s|clientId: ${IRS_KEYCLOAK_CLIENT_ID}|clientId: |g" ${EDC_CONSUMER_VALUES_FILE}; + sed -i "s|clientSecret: ${IRS_KEYCLOAK_CLIENT_SECRET}|clientSecret: |g" ${EDC_CONSUMER_VALUES_FILE}; + + # ssi + sed -i "s|${MIW_URL}||g" ${EDC_CONSUMER_VALUES_FILE}; + sed -i "s|${AUTHORITY_ID}||g" ${EDC_CONSUMER_VALUES_FILE}; + sed -i "s|${SSI_CLIENT_ID}||g" ${EDC_CONSUMER_VALUES_FILE}; + + + ### PROVIDER EDC CONNECTOR + + echo "Replace actual values with placholders in edc-provider values file..." + echo + + sed -i "s|${PARTICIPANT_ID}||g" ${EDC_PROVIDER_VALUES_FILE}; + + + # Postgres DB + sed -i "s|${POSTGRES_DB_USER}||g" ${EDC_PROVIDER_VALUES_FILE}; + sed -i "s|password: ${POSTGRES_DB_PASSWORD}|password: |g" ${EDC_PROVIDER_VALUES_FILE}; + + # Hashicorp vault + sed -i "s|${VAULT_HASHICORP_URL}||g" ${EDC_PROVIDER_VALUES_FILE}; + sed -i "s|${VAULT_HASHICORP_SECRET_PATH}||g" ${EDC_PROVIDER_VALUES_FILE}; + sed -i "s|${VAULT_HASHICORP_TOKEN}||g" ${EDC_PROVIDER_VALUES_FILE}; + + # EDC Oauth + sed -i "s|authKey: ${EDC_API_AUTH_KEY}|authKey: |g" ${EDC_PROVIDER_VALUES_FILE}; + sed -i "s|secret: ${EDC_API_AUTH_KEY}|secret: |g" ${EDC_PROVIDER_VALUES_FILE}; + + # ssi + sed -i "s|${MIW_URL}||g" ${EDC_PROVIDER_VALUES_FILE}; + sed -i "s|${AUTHORITY_ID}||g" ${EDC_PROVIDER_VALUES_FILE}; + sed -i "s|${SSI_CLIENT_ID}||g" ${EDC_PROVIDER_VALUES_FILE}; + + + ### PROVIDER REGISTRY + + echo "Replace secrets with actual values in registry values file..." + echo + + # Postgres DB + sed -i "s|${REGISTRY_USER}||g" ${REGISTRY_VALUES_FILE}; + sed -i "s|password: ${REGISTRY_PASSWORD}|password: |g" ${REGISTRY_VALUES_FILE}; + + +} + + + if [[ "$1" -eq 1 ]] ; then + echo "Setting values in the following helm values (DPP, edc-consumer, edc-provider, registry)" + set_values + elif [[ "$1" -eq 0 ]] ; then + echo "Un Setting values in the following helm values (DPP, edc-consumer, edc-provider, registry)" + unset_values + else + echo "invalid value provided, please provide a value 0 or 1" + fi + +echo "Done..." diff --git a/docs/GETTING-STARTED.md b/docs/GETTING-STARTED.md index d63478710..a04c5a8ea 100644 --- a/docs/GETTING-STARTED.md +++ b/docs/GETTING-STARTED.md @@ -61,13 +61,13 @@ final __representation of the data__. #### Start Minikube Cluster ```bash # start minikube cluster -minikube start +minikube start --cpus 4 --memory 8096 # enable minikube ingress addon minikube addons enable ingress ``` -The secrets/credentials for all components are stored in CX Hashicorp vault (a CatenaX shared service). There is a argocd-vault-plugin which retrieves secrets when it comes to INT or DEV, but the plugin does not work locally as we are not using argocd to deploy the apps in localhost. Therefore, the secrets variables in configurations need to be substituted with their actual values and security must also be ensured during the substitution process. To achieve this, a shell script is used to set [set_dev_values.sh](../deployment/local/scripts/set_dev_values.sh) and unset values [unset_dev_values.sh](../deployment/local/scripts/unset_dev_values.sh) in required components as needed. +The secrets/credentials for all components are stored in CX Hashicorp vault (a CatenaX shared service). There is a argocd-vault-plugin which retrieves secrets when it comes to INT or DEV, but the plugin does not work locally as we are not using argocd to deploy the apps in localhost. Therefore, the secrets variables in configurations need to be substituted with their actual values and security must also be ensured during the substitution process. To achieve this, a shell script is used to set/unset [init-values.sh](../deployment/local/scripts/init-values.sh) in required components as needed. > Prerequisite: Prior to run the scripts, the values for the follwoing environment variables should be placed in the script. @@ -81,10 +81,15 @@ __Script Environment Variables:__ cd ../deployment/local/scripts # set values for local run -./set_dev_values.sh +# ./init-values.sh <0 or 1> +# 0: unset values back to the placeholders +# 1: set actual values from the vault storage + +# set values +./init-values.sh 1 # unset values -./unset_dev_values.sh +./init-values.sh 0 ```