Switch peers from using variable IPs to hostnames.

* Use k8s service hostname for quorum peers: permissioned-nodes.json and static-nodes.json. * Use k8s service hostname in tessera config for peer discovery. * The hostnames are obtained via the quorum services which are known at the time of generation and do not change. This removes the need to do sed and manipulation on the config files after they are deployed in order to set the IPs (which are not known until after k8s resources have been deployed). This also helps with automatically updating the config files when new nodes are deployed or removed, thus avoiding restarting the nodes. * Enabled by this PR: Consensys/quorum#885.
libby · Aug 20, 2020 · 6afd02e · 6afd02e
1 parent b6a6300
commit 6afd02e
Show file tree

Hide file tree

Showing 7 changed files with 19 additions and 48 deletions.
diff --git a/qubernetes b/qubernetes
@@ -112,31 +112,6 @@ end
 # create the output directory if it doesn't exist
 `mkdir -p out`
 
-sed_string = ""
-sed_string_no_escape = ""
-
-# make all services
-# set the replacement string sed command which is run on the permissioned-nodes.json,
-# as the service host IPs of the nodes are not known until they are deployed, and need
-# to be properly set in the permissioned-nodes.json.
-#PERM_NODE_JSON=$(echo $PERM_NODE_TMPL | sed \"s/%QUORUM_DEPLOYMENT_01_SERVICE_HOST%/$QUORUM_NODE01_SERVICE_HOST/g\" | sed \"s/\\$QUORUM_DEPLOYMENT_02_SERVICE_HOST/$QUORUM_NODE02_SERVICE_HOST/g\");
-@nodes.each do |node|
-  set_node_template_vars(node)
-#  puts ("#{@Node_UserIdent}")
-  k8_service_host_var = ("#{@Node_UserIdent}".upcase + "_SERVICE_HOST").gsub("-", "_")
-  permission_node_host="#{@Node_UserIdent}".upcase + "_SERVICE_HOST"
- # puts (k8_service_host_var)
-  sed_instruction = ' sed \"s/%' + permission_node_host + '%/$' + k8_service_host_var + '/g\"'
-  sed_instruction_no_escape = ' sed "s/%' + permission_node_host + '%/$' + k8_service_host_var + '/g"'
-  sed_string = sed_string + sed_instruction + " | "
-  sed_string_no_escape = sed_string_no_escape + sed_instruction_no_escape + " | "
-end
-
-# used/written inside each generated deployment file.
-@Sed_Set_Node_Service_Host = sed_string[0...-2]
-# used inside cofigMap script
-@Sed_Set_Node_Service_Host_No_Escape = sed_string_no_escape[0...-2]
-
 @base_template_path = "templates/k8s"
 #puts (sed_string)
 

diff --git a/templates/k8s/quorum-deployment.yaml.erb b/templates/k8s/quorum-deployment.yaml.erb
@@ -151,13 +151,10 @@ spec:
            CONFIG_TMPL=$(cat ${DDIR}/tessera-config${TESSERA_CONFIG_TYPE}.json.tmpl);
 
 
-           echo  \"<%= @Sed_Set_Node_Service_Host %>\";
-           CONFIG_WITH_OTHER_HOSTS=$(echo $CONFIG_TMPL | <%= @Sed_Set_Node_Service_Host %>) ;
-           CONFIG_WITH_ALL_HOSTS=$(echo $CONFIG_WITH_OTHER_HOSTS | sed \"s/%THIS_SERVICE_HOST%/$<%= @This_Host %>/g\");
            PRIV_KEY=$(cat $DDIR/tm.key)
            PUB_KEY=$(cat $DDIR/tm.pub)
-           CONFIG_FINAL=$(echo $CONFIG_WITH_ALL_HOSTS | sed \"s-%THIS_PRIV_KEY%-${PRIV_KEY}-g\" |  sed \"s-%THIS_PUB_KEY%-${PUB_KEY}-g\");
-           CONFIG_FINAL_9_0=$(echo $CONFIG_WITH_ALL_HOSTS | sed \"s-%THIS_PRIV_KEY%-${DDIR}/tm.key-g\" |  sed \"s-%THIS_PUB_KEY%-${DDIR}/tm.pub-g\");
+           CONFIG_FINAL=$(echo $CONFIG_TMPL | sed \"s-%THIS_PRIV_KEY%-${PRIV_KEY}-g\" |  sed \"s-%THIS_PUB_KEY%-${PUB_KEY}-g\");
+           CONFIG_FINAL_9_0=$(echo $CONFIG_TMPL | sed \"s-%THIS_PRIV_KEY%-${DDIR}/tm.key-g\" |  sed \"s-%THIS_PUB_KEY%-${DDIR}/tm.pub-g\");
            if [[ \"${Tess_Ver_Second}\" -ge \"9\" ]]; then CONFIG_FINAL=${CONFIG_FINAL_9_0}; fi;
            echo $CONFIG_FINAL >  ${DDIR}/tessera-config-with-hosts.json;
            cat  ${DDIR}/tessera-config-with-hosts.json;
@@ -345,7 +342,7 @@ spec:
           name: quorum-permissioned-config
           items:
           - key: permissioned-nodes.json
-            path: permissioned-nodes.json.tmpl
+            path: permissioned-nodes.json
           - key: permissioned-update.sh
             path: permissioned-update.sh
       - name: genesis-config-persistent-storage

diff --git a/templates/k8s/quorum-shared-config.yaml.erb b/templates/k8s/quorum-shared-config.yaml.erb
@@ -16,9 +16,7 @@ data:
 
   permissioned-update.sh: |
     #!/bin/bash
-    PERM_NODE_TMPL=$(cat $QHOME/permission-nodes/permissioned-nodes.json.tmpl);
-    PERM_NODE_JSON=$(echo $PERM_NODE_TMPL | <%= @Sed_Set_Node_Service_Host_No_Escape %>);
-    echo $PERM_NODE_JSON >  $QUORUM_DATA_DIR/permissioned-nodes.json;
+    cp $QUORUM_HOME/permission-nodes/permissioned-nodes.json $QUORUM_DATA_DIR/permissioned-nodes.json;
     cp $QUORUM_DATA_DIR/permissioned-nodes.json $QUORUM_DATA_DIR/static-nodes.json;
 
 ---

diff --git a/templates/quorum/permissioned-nodes.json.erb b/templates/quorum/permissioned-nodes.json.erb
@@ -9,10 +9,11 @@ end
    # The enode value is obtained from that file and set in the permissioned-nodes.json
 -%>
 [
-<%- @nodes.each_with_index do |node, indexNode| -%>
-    <%- set_node_template_vars(node) -%>
-    <%- @Enode_File = @Key_Dir_Base + "/" + @Node_Key_Dir + "/enode" -%>
-    <%- File.readlines(@Enode_File).each do |line| @Enode = "#{line}".gsub(/\s+/, "") end -%>
-  "enode://<%= @Enode %>@%<%= "#{@Node_UserIdent}".upcase %>_SERVICE_HOST%:<%= @NodeP2P_ListenAddr%>?discport=0&raftport=<%= @Raft_Port%>"<%- if (indexNode != @nodes.size - 1) %>,<%- end %>
-<% end -%>
+  <%- @nodes.each_with_index do |node, indexNode| -%>
+  <%- set_node_template_vars(node) -%>
+  <%- @Enode_File = @Key_Dir_Base + "/" + @Node_Key_Dir + "/enode" -%>
+  <%- File.readlines(@Enode_File).each do |line| @Enode = "#{line}".gsub(/\s+/, "") end -%>
+  <%- # use the service name / kube-dns to look up the nodes -%>
+  "enode://<%= @Enode %>@<%= "#{@Node_UserIdent}"%>:<%= @NodeP2P_ListenAddr%>?discport=0&raftport=<%= @Raft_Port%>"<%- if (indexNode != @nodes.size - 1) %>,<%- end %>
+  <% end -%>
 ]
diff --git a/templates/quorum/tessera-config-9.0.json.erb b/templates/quorum/tessera-config-9.0.json.erb
@@ -24,7 +24,7 @@ end
   {
     "app":"ThirdParty",
     "enabled": true,
-    "serverAddress": "http://%THIS_SERVICE_HOST%:<%= @Tm_3Party_Port %>",
+    "serverAddress": "http://<%= @Node_UserIdent %>:<%= @Tm_3Party_Port %>",
     "cors": {
         "allowedMethods": [
             "GET",
@@ -45,7 +45,7 @@ end
   {
     "app":"P2P",
     "enabled": true,
-    "serverAddress":"http://%THIS_SERVICE_HOST%:<%= @Tm_Port %>",
+    "serverAddress":"http://<%= @Node_UserIdent %>:<%= @Tm_Port %>",
     "sslConfig": {
       "tls": "OFF",
       "generateKeyStoreIfNotExisted": true,
@@ -70,7 +70,7 @@ end
 <%- @nodes.each_with_index do |node, indexNode| %>
 <%= set_node_template_vars(node) -%>
      {
-           "url": "http://%<%= "#{@Node_UserIdent}".upcase %>_SERVICE_HOST%:<%= @Tm_Port %>"
+           "url": "http://<%= @Node_UserIdent %>:<%= @Tm_Port %>"
          }<%- if (indexNode != @nodes.size - 1) %>,<%- end %>
 
 <% end -%>

diff --git a/templates/quorum/tessera-config-enhanced.json.erb b/templates/quorum/tessera-config-enhanced.json.erb
@@ -27,7 +27,7 @@ end
     "serverSocket":{
       "type":"INET",
       "port": <%= @Tm_3Party_Port %>,
-      "hostName": "http://%THIS_SERVICE_HOST%"
+      "hostName": "http://<%= @Node_UserIdent %>"
     },
     "communicationType" : "REST"
   },
@@ -46,7 +46,7 @@ end
     "serverSocket":{
       "type":"INET",
       "port": <%= @Tm_Port %>,
-      "hostName": "http://%THIS_SERVICE_HOST%",
+      "hostName": "http://<%= @Node_UserIdent %>",
       "bindingAddress": "http://0.0.0.0:<%= @Tm_Port %>"
     },
     "sslConfig": {
@@ -73,7 +73,7 @@ end
 <%- @nodes.each_with_index do |node, indexNode| %>
 <%= set_node_template_vars(node) -%>
      {
-           "url": "http://%<%= "#{@Node_UserIdent}".upcase %>_SERVICE_HOST%:<%= @Tm_Port %>"
+           "url": "http://<%= @Node_UserIdent %>:<%= @Tm_Port %>"
          }<%- if (indexNode != @nodes.size - 1) %>,<%- end %>
 
 <% end -%>

diff --git a/templates/quorum/tessera-config.json.erb b/templates/quorum/tessera-config.json.erb
@@ -23,7 +23,7 @@ end
     },
     "server": {
         "port": <%= @Tm_Port %>,
-        "hostName": "http://%THIS_SERVICE_HOST%",
+        "hostName": "http://<%= @Node_UserIdent %>",
         "bindingAddress": "http://0.0.0.0:<%= @Tm_Port %>",
         "sslConfig": {
             "tls": "OFF",
@@ -46,7 +46,7 @@ end
 <%- @nodes.each_with_index do |node, indexNode| %>
     <%= set_node_template_vars(node) -%>
      {
-           "url": "http://%<%= "#{@Node_UserIdent}".upcase %>_SERVICE_HOST%:<%= @Tm_Port %>"
+           "url": "http://%<%= @Node_UserIdent%>%:<%= @Tm_Port %>"
          }<%- if (indexNode != @nodes.size - 1) %>,<%- end %>
 
 <% end -%>