From 5afa03fd9e28ccab4ea98f12a565c6f78a101f06 Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Sun, 22 Oct 2023 22:20:22 -0700 Subject: [PATCH 1/7] Support for Signer Authorized Keys Authorized key is the tezos native method to authenticate signing requests, one that we use in the new tezos-kms-signer-lambda. This adds the required support on tezos-k8s to sign with such a signer. The way it works in octez is: * when the baker/client connects to the signer for the first time, signer answers with a list of "authorized_keys" that the signature request must be signed with. These authorized keys are just tezos accounts * if the baker/client has the secret key for one of these authorized keys, they will just sign every request with it. otherwise, there will be an error * this can't be nested. the authorized_key can't be remote We add support in tezos-k8s by assuming the authorized_keys are just standard "accounts". Then, you may configure a baker as follows: ``` nodes: mybaker: bake_using_accounts: - mybakeraddy authorized_keys: - my_authorized_key ``` config-generator then ensures that the private authorized key is accessible to the baker. We also add support on octez-signer end: ``` octezSigners: mysigner: sign_for_accounts: - mybakeraddy authorized_keys: - my_authorized_key ``` When set, the signer mandates requests to be authenticated. Otherwise, it signs anything. This way, you can test end-to-end in a private chain. We modify mkchain to do this by default: mkchain now generates an authorized key and uses it to sign by default. Also, mkchain was previously defaulting to using one remote signer, but this broke when adding support for tacoInfra signer. I fixed it. I have tested it with 3 bakers and 2 signers, one authorized and one not. It's all working. I haven't tried zerotier and public chains. Other changes: * switch default version to 17.3 * no magic byte restriction from signer - prevents activation --- charts/tezos/scripts/remote-signer.sh | 6 +++++- charts/tezos/values.yaml | 6 +++++- mkchain/README.md | 2 +- mkchain/tqchain/mkchain.py | 14 ++++++++------ test/charts/mainnet.expect.yaml | 10 +++++----- test/charts/mainnet2.expect.yaml | 10 +++++----- test/charts/private-chain.expect.yaml | 6 +++++- utils/config-generator.py | 19 ++++++++++++++++++- 8 files changed, 52 insertions(+), 21 deletions(-) diff --git a/charts/tezos/scripts/remote-signer.sh b/charts/tezos/scripts/remote-signer.sh index 5fe4d0b25..53deb6af8 100644 --- a/charts/tezos/scripts/remote-signer.sh +++ b/charts/tezos/scripts/remote-signer.sh @@ -6,7 +6,11 @@ CLIENT_DIR="$TEZ_VAR/client" NODE_DIR="$TEZ_VAR/node" NODE_DATA_DIR="$TEZ_VAR/node/data" -CMD="$TEZ_BIN/octez-signer -d $CLIENT_DIR launch http signer --magic-bytes 0x11,0x12,0x13 --check-high-watermark -a 0.0.0.0 -p 6732" +extra_args="" +if [ -f ${CLIENT_DIR}/authorized_keys ]; then + extra_args="${extra_args} --require-authentication" +fi +CMD="$TEZ_BIN/octez-signer -d $CLIENT_DIR ${extra_args} launch http signer -a 0.0.0.0 -p 6732" # ensure we can run tezos-signer commands without specifying client dir ln -s /var/tezos/client /home/tezos/.tezos-signer diff --git a/charts/tezos/values.yaml b/charts/tezos/values.yaml index 144918155..c863ac2da 100644 --- a/charts/tezos/values.yaml +++ b/charts/tezos/values.yaml @@ -8,7 +8,7 @@ is_invitation: false # Images not part of the tezos-k8s repo go here images: - octez: tezos/tezos:v17.1 + octez: tezos/tezos:v17.3 tacoinfraRemoteSigner: ghcr.io/oxheadalpha/tacoinfra-remote-signer:0.1.0 # Images that are part of the tezos-k8s repo go here with 'dev' tag tezos_k8s_images: @@ -174,6 +174,7 @@ should_generate_unsafe_deterministic_data: false # Don't also set `bake_using_accounts`. # - `bake_using_accounts`: List of account names that should be used for baking. # Don't also set `bake_using_account`. +# - `authorized_keys`: List of account names available to the baker to sign signature requests. # - `config`: Same as the outer statefulset level `config`. It overrides the # statefulset level. # - `is_bootstrap_node`: Boolean for is this node a bootstrap peer. @@ -314,6 +315,9 @@ octezSigners: {} # tezos-signer-0: # accounts: # - baker0 +# authorized_keys: +# # if set, baker will only sign request authenticated by one of the authorized_keys +# - authorized_key0 # ``` # # Deploys a signer using AWS KMS to sign operations. diff --git a/mkchain/README.md b/mkchain/README.md index b408fc441..6cfde391e 100644 --- a/mkchain/README.md +++ b/mkchain/README.md @@ -86,7 +86,7 @@ You can explicitly specify some values by: | | --number-of-nodes | Number of non-baking nodes in the cluster | 0 | | bootstrap_peers | --bootstrap-peers | Peer ips to connect to | [] | | expected_proof_of_work | --expected-proof-of-work | Node identity generation difficulty | 0 | -| images.octez | --octez-docker-image | Version of the Octez docker image to run | tezos/tezos:v17.1 | +| images.octez | --octez-docker-image | Version of the Octez docker image to run | tezos/tezos:v17.3 | | | --use-docker (--no...) | Use (or don't use) docker to generate keys rather than pytezos | autodetect | | zerotier_config.zerotier_network | --zerotier-network | Zerotier network id for external chain access | | | zerotier_config.zerotier_token | --zerotier-token | Zerotier token for external chain access | | diff --git a/mkchain/tqchain/mkchain.py b/mkchain/tqchain/mkchain.py index dd2c7daf6..6030945ce 100644 --- a/mkchain/tqchain/mkchain.py +++ b/mkchain/tqchain/mkchain.py @@ -70,7 +70,7 @@ def quoted_scalar(dumper, data): # a representer to force quotations on scalars }, "octez_docker_image": { "help": "Version of the Octez docker image", - "default": "tezos/tezos:v17.1", + "default": "tezos/tezos:v17.3", }, "use_docker": { "action": "store_true", @@ -154,6 +154,7 @@ def node_config(name, n, is_baker): "shell": {"history_mode": "rolling"}, "metrics_addr": [":9932"], }, + "authorized_keys": ["authorized-key-0"], } if is_baker: ret["bake_using_accounts"] = [f"{name}-{n}"] @@ -243,7 +244,7 @@ def main(): baking_accounts = { f"{ARCHIVE_BAKER_NODE_NAME}-{n}": {} for n in range(args.number_of_bakers) } - for account in baking_accounts: + for account in [*baking_accounts, *["authorized-key-0"]]: print(f"Generating keys for account {account}") keys = gen_key(args.octez_docker_image) for key_type in keys: @@ -275,11 +276,12 @@ def main(): ], } - signers = { + octezSigners = { "tezos-signer-0": { - "sign_for_accounts": [ + "accounts": [ f"{ARCHIVE_BAKER_NODE_NAME}-{n}" for n in range(args.number_of_bakers) - ] + ], + "authorized_keys": ["authorized-key-0"], } } @@ -308,7 +310,7 @@ def main(): **base_constants, "bootstrap_peers": bootstrap_peers, "accounts": accounts["secret"], - "signers": signers, + "octezSigners": octezSigners, "nodes": creation_nodes, **activation, } diff --git a/test/charts/mainnet.expect.yaml b/test/charts/mainnet.expect.yaml index 9264f9117..6fc41d610 100644 --- a/test/charts/mainnet.expect.yaml +++ b/test/charts/mainnet.expect.yaml @@ -38,7 +38,7 @@ data: ARCHIVE_TARBALL_URL: "" PREFER_TARBALLS: "false" SNAPSHOT_SOURCE: "https://xtz-shots.io/tezos-snapshots.json" - OCTEZ_VERSION: "tezos/tezos:v17.1" + OCTEZ_VERSION: "tezos/tezos:v17.3" NODE_GLOBALS: | { "config": {}, @@ -128,7 +128,7 @@ spec: spec: containers: - name: octez-node - image: "tezos/tezos:v17.1" + image: "tezos/tezos:v17.3" imagePullPolicy: IfNotPresent command: - /bin/sh @@ -213,7 +213,7 @@ spec: memory: 80Mi initContainers: - name: config-init - image: "tezos/tezos:v17.1" + image: "tezos/tezos:v17.3" imagePullPolicy: IfNotPresent command: - /bin/sh @@ -324,7 +324,7 @@ spec: - mountPath: /var/tezos name: var-volume - name: snapshot-importer - image: "tezos/tezos:v17.1" + image: "tezos/tezos:v17.3" imagePullPolicy: IfNotPresent command: - /bin/sh @@ -387,7 +387,7 @@ spec: - mountPath: /var/tezos name: var-volume - name: upgrade-storage - image: "tezos/tezos:v17.1" + image: "tezos/tezos:v17.3" imagePullPolicy: IfNotPresent command: - /bin/sh diff --git a/test/charts/mainnet2.expect.yaml b/test/charts/mainnet2.expect.yaml index 7497c1f39..ccc8c09b3 100644 --- a/test/charts/mainnet2.expect.yaml +++ b/test/charts/mainnet2.expect.yaml @@ -38,7 +38,7 @@ data: ARCHIVE_TARBALL_URL: "" PREFER_TARBALLS: "false" SNAPSHOT_SOURCE: "https://xtz-shots.io/tezos-snapshots.json" - OCTEZ_VERSION: "tezos/tezos:v17.1" + OCTEZ_VERSION: "tezos/tezos:v17.3" NODE_GLOBALS: | { "config": {}, @@ -195,7 +195,7 @@ spec: spec: containers: - name: octez-node - image: "tezos/tezos:v17.1" + image: "tezos/tezos:v17.3" imagePullPolicy: IfNotPresent command: - /bin/sh @@ -316,7 +316,7 @@ spec: memory: 80Mi initContainers: - name: config-init - image: "tezos/tezos:v17.1" + image: "tezos/tezos:v17.3" imagePullPolicy: IfNotPresent command: - /bin/sh @@ -433,7 +433,7 @@ spec: - mountPath: /var/tezos name: var-volume - name: snapshot-importer - image: "tezos/tezos:v17.1" + image: "tezos/tezos:v17.3" imagePullPolicy: IfNotPresent command: - /bin/sh @@ -498,7 +498,7 @@ spec: - mountPath: /var/tezos name: var-volume - name: upgrade-storage - image: "tezos/tezos:v17.1" + image: "tezos/tezos:v17.3" imagePullPolicy: IfNotPresent command: - /bin/sh diff --git a/test/charts/private-chain.expect.yaml b/test/charts/private-chain.expect.yaml index fd60278d9..3d3b8fe1b 100644 --- a/test/charts/private-chain.expect.yaml +++ b/test/charts/private-chain.expect.yaml @@ -1561,7 +1561,11 @@ spec: NODE_DIR="$TEZ_VAR/node" NODE_DATA_DIR="$TEZ_VAR/node/data" - CMD="$TEZ_BIN/octez-signer -d $CLIENT_DIR launch http signer --magic-bytes 0x11,0x12,0x13 --check-high-watermark -a 0.0.0.0 -p 6732" + extra_args="" + if [ -f ${CLIENT_DIR}/authorized_keys ]; then + extra_args="${extra_args} --require-authentication" + fi + CMD="$TEZ_BIN/octez-signer -d $CLIENT_DIR ${extra_args} launch http signer -a 0.0.0.0 -p 6732" # ensure we can run tezos-signer commands without specifying client dir ln -s /var/tezos/client /home/tezos/.tezos-signer diff --git a/utils/config-generator.py b/utils/config-generator.py index 35f1c832c..907caee94 100755 --- a/utils/config-generator.py +++ b/utils/config-generator.py @@ -332,6 +332,11 @@ def expose_secret_key(account_name): pod. It returns the obvious Boolean. """ if MY_POD_TYPE == "activating": + all_authorized_keys = [key for node in NODES.values() for instance in node['instances'] for key in instance.get('authorized_keys', [])] + if account_name in all_authorized_keys: + # populate all known authorized keys in the activation account. + # This avoids annoying edge cases for activating private chains, when security is not critical. + return True return NETWORK_CONFIG["activation_account_name"] == account_name if MY_POD_TYPE == "signing": @@ -340,6 +345,8 @@ def expose_secret_key(account_name): if MY_POD_TYPE == "node": if MY_POD_CONFIG.get("bake_using_account", "") == account_name: return True + if account_name in MY_POD_CONFIG.get("authorized_keys", {}): + return True return account_name in MY_POD_CONFIG.get("bake_using_accounts", {}) return False @@ -419,6 +426,7 @@ def import_keys(all_accounts): secret_keys = [] public_keys = [] public_key_hashs = [] + authorized_keys = [] for account_name, account_values in all_accounts.items(): print("\n Importing keys for account: " + account_name) @@ -453,6 +461,11 @@ def import_keys(all_accounts): public_key_hashs.append({"name": account_name, "value": pkh_b58}) account_values["pkh"] = pkh_b58 + if MY_POD_TYPE == "signing" and \ + account_name in MY_POD_CONFIG.get("authorized_keys", {}): + print(f" Appending authorized key: {pk_b58}") + authorized_keys.append({"name": account_name, "value": pk_b58}) + print(f" Account key type: {account_values.get('type')}") print( f" Account bootstrap balance: " @@ -463,10 +476,11 @@ def import_keys(all_accounts): + f"{account_values.get('is_bootstrap_baker_account', False)}" ) - sk_path, pk_path, pkh_path = ( + sk_path, pk_path, pkh_path, ak_path = ( f"{tezdir}/secret_keys", f"{tezdir}/public_keys", f"{tezdir}/public_key_hashs", + f"{tezdir}/authorized_keys", ) print(f"\n Writing {sk_path}") json.dump(secret_keys, open(sk_path, "w"), indent=4) @@ -474,6 +488,9 @@ def import_keys(all_accounts): json.dump(public_keys, open(pk_path, "w"), indent=4) print(f" Writing {pkh_path}") json.dump(public_key_hashs, open(pkh_path, "w"), indent=4) + if MY_POD_TYPE == "signing" and len(authorized_keys) > 0: + print(f" Writing {ak_path}") + json.dump(authorized_keys, open(ak_path, "w"), indent=4) def create_node_identity_json(): From 804b2c8503880246b32850f7213818301617d7cf Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Tue, 24 Oct 2023 21:14:03 -0700 Subject: [PATCH 2/7] Update mkchain/tqchain/mkchain.py Co-authored-by: Aryeh Harris --- mkchain/tqchain/mkchain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkchain/tqchain/mkchain.py b/mkchain/tqchain/mkchain.py index 6030945ce..da8364e44 100644 --- a/mkchain/tqchain/mkchain.py +++ b/mkchain/tqchain/mkchain.py @@ -244,7 +244,7 @@ def main(): baking_accounts = { f"{ARCHIVE_BAKER_NODE_NAME}-{n}": {} for n in range(args.number_of_bakers) } - for account in [*baking_accounts, *["authorized-key-0"]]: + for account in [*baking_accounts, "authorized-key-0"]: print(f"Generating keys for account {account}") keys = gen_key(args.octez_docker_image) for key_type in keys: From aa9233a84bcaff2b9f433e107fd6d9c3acdfb096 Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Tue, 24 Oct 2023 21:14:22 -0700 Subject: [PATCH 3/7] Update charts/tezos/values.yaml Co-authored-by: Aryeh Harris --- charts/tezos/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/tezos/values.yaml b/charts/tezos/values.yaml index c863ac2da..e0c7377d7 100644 --- a/charts/tezos/values.yaml +++ b/charts/tezos/values.yaml @@ -317,7 +317,7 @@ octezSigners: {} # - baker0 # authorized_keys: # # if set, baker will only sign request authenticated by one of the authorized_keys -# - authorized_key0 +# - authorized-key-0 # ``` # # Deploys a signer using AWS KMS to sign operations. From e1c5304afc12602b262369928229516a5d476253 Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Tue, 24 Oct 2023 21:29:54 -0700 Subject: [PATCH 4/7] comment phrasing, per review --- charts/tezos/values.yaml | 10 ++++++++-- utils/config-generator.py | 21 +++++++++++++++------ 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/charts/tezos/values.yaml b/charts/tezos/values.yaml index e0c7377d7..84afa8df0 100644 --- a/charts/tezos/values.yaml +++ b/charts/tezos/values.yaml @@ -174,7 +174,10 @@ should_generate_unsafe_deterministic_data: false # Don't also set `bake_using_accounts`. # - `bake_using_accounts`: List of account names that should be used for baking. # Don't also set `bake_using_account`. -# - `authorized_keys`: List of account names available to the baker to sign signature requests. +# - `authorized_keys`: Keys used to authenticate a baker to a signer. +# When a baker uses a remote signer that requires +# authentication, the relevant key from this list +# will be used to sign every signature request. # - `config`: Same as the outer statefulset level `config`. It overrides the # statefulset level. # - `is_bootstrap_node`: Boolean for is this node a bootstrap peer. @@ -316,7 +319,10 @@ octezSigners: {} # accounts: # - baker0 # authorized_keys: -# # if set, baker will only sign request authenticated by one of the authorized_keys +# # Keys used to authenticate the baker to the signer. +# # The baker must have the private key for one of the +# # listed accounts. The signer will only sign a request +# # from a baker authenticated by an allowed key. # - authorized-key-0 # ``` # diff --git a/utils/config-generator.py b/utils/config-generator.py index 907caee94..46f403c5a 100755 --- a/utils/config-generator.py +++ b/utils/config-generator.py @@ -332,10 +332,16 @@ def expose_secret_key(account_name): pod. It returns the obvious Boolean. """ if MY_POD_TYPE == "activating": - all_authorized_keys = [key for node in NODES.values() for instance in node['instances'] for key in instance.get('authorized_keys', [])] + all_authorized_keys = [ + key + for node in NODES.values() + for instance in node["instances"] + for key in instance.get("authorized_keys", []) + ] if account_name in all_authorized_keys: - # populate all known authorized keys in the activation account. - # This avoids annoying edge cases for activating private chains, when security is not critical. + # Populate authorized keys known by all bakers in the activation account. + # This ensures that activation will succeed with a remote signer that requires auth, + # regardless of which baker does it. return True return NETWORK_CONFIG["activation_account_name"] == account_name @@ -461,8 +467,9 @@ def import_keys(all_accounts): public_key_hashs.append({"name": account_name, "value": pkh_b58}) account_values["pkh"] = pkh_b58 - if MY_POD_TYPE == "signing" and \ - account_name in MY_POD_CONFIG.get("authorized_keys", {}): + if MY_POD_TYPE == "signing" and account_name in MY_POD_CONFIG.get( + "authorized_keys", {} + ): print(f" Appending authorized key: {pk_b58}") authorized_keys.append({"name": account_name, "value": pk_b58}) @@ -756,7 +763,9 @@ def create_node_snapshot_config_json(history_mode): ] if octez_version: matching_snapshots = [ - s for s in matching_snapshots if int(octez_version) == s.get("tezos_version").get("version").get("major") + s + for s in matching_snapshots + if int(octez_version) == s.get("tezos_version").get("version").get("major") ] matching_snapshots = sorted(matching_snapshots, key=lambda s: s.get("block_height")) From 2ec0c45198b3c6e111a5db3a914f8f96d09e842b Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Sun, 5 Nov 2023 20:16:00 -0800 Subject: [PATCH 5/7] fix comments per review --- charts/tezos/values.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/charts/tezos/values.yaml b/charts/tezos/values.yaml index 84afa8df0..f8f5b0f94 100644 --- a/charts/tezos/values.yaml +++ b/charts/tezos/values.yaml @@ -174,7 +174,8 @@ should_generate_unsafe_deterministic_data: false # Don't also set `bake_using_accounts`. # - `bake_using_accounts`: List of account names that should be used for baking. # Don't also set `bake_using_account`. -# - `authorized_keys`: Keys used to authenticate a baker to a signer. +# - `authorized_keys`: List of account names that should be used as keys to +# authenticate a baker to a signer. # When a baker uses a remote signer that requires # authentication, the relevant key from this list # will be used to sign every signature request. @@ -319,10 +320,10 @@ octezSigners: {} # accounts: # - baker0 # authorized_keys: -# # Keys used to authenticate the baker to the signer. -# # The baker must have the private key for one of the -# # listed accounts. The signer will only sign a request -# # from a baker authenticated by an allowed key. +# # Names of accounts used to authenticate the baker to the signer. +# # The baker must have the private key for one of the listed +# # accounts. The signer will only sign a request from a baker +# # authenticated by an allowed key. # - authorized-key-0 # ``` # From fd58c598348069900f1337d4fe765dfd8e60efae Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Sun, 5 Nov 2023 20:48:55 -0800 Subject: [PATCH 6/7] validate in helm that authroized keys exist --- charts/tezos/templates/_helpers.tpl | 28 ++++++++++++++++++++++++++++ charts/tezos/templates/configs.yaml | 1 + 2 files changed, 29 insertions(+) diff --git a/charts/tezos/templates/_helpers.tpl b/charts/tezos/templates/_helpers.tpl index f24b0c556..eae8cd4f5 100644 --- a/charts/tezos/templates/_helpers.tpl +++ b/charts/tezos/templates/_helpers.tpl @@ -207,3 +207,31 @@ metadata: {{- end }} {{- "true" }} {{- end }} + +{{/* + Get list of authorized keys. Fails if any of the keys is not defined in the accounts. +*/}} +{{- define "tezos.getAuthorizedKeys" }} + {{- $allAuthorizedKeys := list }} + {{- /* Gather keys from nodes */}} + {{- range $node := .Values.nodes }} + {{- range $instance := $node.instances }} + {{- if .authorized_keys }} + {{- $allAuthorizedKeys = concat $allAuthorizedKeys .authorized_keys }} + {{- end }} + {{- end }} + {{- end }} + {{- /* Gather keys from octezSigners */}} + {{- range $signer := .Values.octezSigners }} + {{- if $signer.authorized_keys }} + {{- $allAuthorizedKeys = concat $allAuthorizedKeys $signer.authorized_keys }} + {{- end }} + {{- end }} + {{- /* Ensure all keys are defined in accounts and fail otherwise */}} + {{- $allAuthorizedKeys = uniq $allAuthorizedKeys }} + {{- range $key := $allAuthorizedKeys }} + {{- if not (index $.Values.accounts $key) }} + {{- fail (printf "Authorized key '%s' is not defined in accounts." $key) }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/tezos/templates/configs.yaml b/charts/tezos/templates/configs.yaml index 6b028dac7..aeb9f9589 100644 --- a/charts/tezos/templates/configs.yaml +++ b/charts/tezos/templates/configs.yaml @@ -114,3 +114,4 @@ metadata: namespace: {{ .Release.Namespace }} --- {{- end }} +{{- include "tezos.getAuthorizedKeys" . }} From be05447b220ae398dd46c762bc92ab350823920c Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Mon, 6 Nov 2023 07:28:59 -0800 Subject: [PATCH 7/7] Update charts/tezos/templates/_helpers.tpl Co-authored-by: Aryeh Harris --- charts/tezos/templates/_helpers.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/tezos/templates/_helpers.tpl b/charts/tezos/templates/_helpers.tpl index eae8cd4f5..4c20ab2f5 100644 --- a/charts/tezos/templates/_helpers.tpl +++ b/charts/tezos/templates/_helpers.tpl @@ -230,7 +230,7 @@ metadata: {{- /* Ensure all keys are defined in accounts and fail otherwise */}} {{- $allAuthorizedKeys = uniq $allAuthorizedKeys }} {{- range $key := $allAuthorizedKeys }} - {{- if not (index $.Values.accounts $key) }} + {{- if not (index $.Values.accounts $key "key") }} {{- fail (printf "Authorized key '%s' is not defined in accounts." $key) }} {{- end }} {{- end }}