diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3b44222 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +gen-node-keys.sh +out/*.yaml +out/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..0b2236e --- /dev/null +++ b/README.md @@ -0,0 +1,71 @@ +## Qubernetes + +A project for deployng [Quorum](https://github.com/jpmorganchase/quorum) on [Kubernetes](https://github.com/kubernetes/kubernetes). + +Thanks to https://medium.com/@cryptoctl which provided +and awesome starting point. + +## Quick Start +* Set up the intial config in `qubernetes.yaml` + +1. Genearte the kubernetes resource yaml files required + for a Quorum deployment. +``` +# Generate the keys, permissioned-nodes.json file +# genesis.json for the configured nodes +$> ./quorum-init + +# Generate the kubernetes resources +# necessary to support a Quorum deploy +# this will be written to the `out` dir. +$> ./qubernetes + +``` +2. Deploy to kubernetes + +* see helper scripts `deploy.sh` + +``` +kubectl apply -f out/quorum-shared-config.yaml +kubectl apply -f out/quorum-services.yaml +kubectl apply -f out/quorum-keyconfigs.yaml +kubectl apply -f out/quorum-deployments.yaml +``` + + +3. Accessing your nodes + +``` +local $> kubectl get pods --namespace=$YOUR_NAMESPACE +local $> kubect exec -it $POD_ID -c quorum /bin/ash +quorum-qubernetes $> cd /etc/quorum/qdata +quorum-qubernetes $> ls +quorum-qubernetes $> geth attach dd/geth.ipc +> eth.blockNumber +> 0 +> exit + +quorum-qubernetes $> cd /etc/quorum/qdata/contracts +quorum-qubernetes $>./runscript.js public_contract.js + +# you should know see the tx go through +quorum-qubernetes $> geth attach /etc/quorum/qdata/dd/geth.ipc +> eth.blockNumber +> 1 + +# show connected peers +> admin.peers + +``` + + +3. Deleting the deployment + +* see helper scripts `deploy.sh` + +``` +kubectl delete -f out/quorum-shared-config.yaml +kubectl delete -f out/quorum-services.yaml +kubectl delete -f out/quorum-keyconfigs.yaml +kubectl delete -f out/quorum-deployments.yaml +``` diff --git a/config/qubernetes-7.yaml b/config/qubernetes-7.yaml new file mode 100644 index 0000000..1b5df63 --- /dev/null +++ b/config/qubernetes-7.yaml @@ -0,0 +1,164 @@ +namespace: + name: quorum-test + +# here you can add as many nodes as you like, name and configure them +# Note: +# 1. need to be in the list of permissioned nodes and static nodes. +# 2. keys should be set locally. +nodes: +- member: + Node_UserIdent: quorum-node01 + Key_Dir: key1 + # create a private key and add it to the keystore folder + # ... or just use the example one for testing + keystore: + name: key1 + # true: upload secret first via 'kubectl create secret generic geth-key --from-file /path/to/keyfile' + # false: use the key in keystore folder + #secret: false + # quorum transaction manager keys + tm_key: + name: tm1 + nodekey: + name: nodekey1 + +# keep adding nodes +# - member: +# ... +- member: + Node_UserIdent: quorum-node02 + Key_Dir: key2 + # create a private key and add it to the keystore folder + # ... or just use the example one for testing + keystore: + name: key2 + # true: upload secret first via 'kubectl create secret generic geth-key --from-file /path/to/keyfile' + # false: use the key in keystore folder + secret: false + # quorum transaction manager keys + tm_key: + name: tm2 + nodekey: + name: nodekey2 +- member: + Node_UserIdent: quorum-node03 + Key_Dir: key3 + # create a private key and add it to the keystore folder + # ... or just use the example one for testing + keystore: + name: key3 + # true: upload secret first via 'kubectl create secret generic geth-key --from-file /path/to/keyfile' + # false: use the key in keystore folder + secret: false + # quorum transaction manager keys + tm_key: + name: tm3 + nodekey: + name: nodekey3 +- member: + Node_UserIdent: quorum-node04 + Key_Dir: key4 + # create a private key and add it to the keystore folder + # ... or just use the example one for testing + keystore: + name: key4 + # true: upload secret first via 'kubectl create secret generic geth-key --from-file /path/to/keyfile' + # false: use the key in keystore folder + secret: false + # quorum transaction manager keys + tm_key: + name: tm4 + nodekey: + name: nodekey4 +- member: + Node_UserIdent: quorum-node05 + Key_Dir: key5 + # create a private key and add it to the keystore folder + # ... or just use the example one for testing + keystore: + name: key5 + # true: upload secret first via 'kubectl create secret generic geth-key --from-file /path/to/keyfile' + # false: use the key in keystore folder + secret: false + # quorum transaction manager keys + tm_key: + name: tm5 + nodekey: + name: nodekey5 +- member: + Node_UserIdent: quorum-node06 + Key_Dir: key6 + # create a private key and add it to the keystore folder + # ... or just use the example one for testing + keystore: + name: key6 + # true: upload secret first via 'kubectl create secret generic geth-key --from-file /path/to/keyfile' + # false: use the key in keystore folder + secret: false + # quorum transaction manager keys + tm_key: + name: tm6 + nodekey: + name: nodekey6 +- member: + Node_UserIdent: quorum-node07 + Key_Dir: key7 + # create a private key and add it to the keystore folder + # ... or just use the example one for testing + keystore: + name: key7 + # true: upload secret first via 'kubectl create secret generic geth-key --from-file /path/to/keyfile' + # false: use the key in keystore folder + secret: false + # quorum transaction manager keys + tm_key: + name: tm7 + nodekey: + name: nodekey7 +- member: + Node_UserIdent: quorum-node08 + Key_Dir: key8 + # create a private key and add it to the keystore folder + # ... or just use the example one for testing + keystore: + name: key8 + # true: upload secret first via 'kubectl create secret generic geth-key --from-file /path/to/keyfile' + # false: use the key in keystore folder + secret: false + # quorum transaction manager keys + tm_key: + name: tm8 + nodekey: + name: nodekey8 +quorum: + # base quorum data dir as set inside each container. + Node_DataDir: /etc/quorum/qdata + # This is where all the keys are store, and/or where they are generated, as in the case of quorum-keygen. + # Either full or relative paths on the machine generating the config + Key_Dir_Base: out/config + Permissioned_Nodes_File: out/config/permissioned-nodes.json + Genesis_File: out/config/genesis.json + # related to quorum containers + quorum: + Raft_Port: 50401 + # container images at https://hub.docker.com/u/quorumengineering/ + Quorum_Version: 2.1.1 + # related to transaction manager containers + tm: + # container images at https://hub.docker.com/u/quorumengineering/ + # TODO: add support for tessera, this should hold the full container name + # in that case. + Tm_Version: 0.3.2 + Port: 9001 + +# generic geth related options +geth: + Node_WSPort: 8546 + NodeP2P_ListenAddr: 21000 + network: + # network id (1: mainnet, 3: ropsten, 4: rinkeby ... ) + id: 1101 + # public (true|false) is it a public network? + public: false + # general verbosity of geth [1..5] + verbosity: 9 diff --git a/config/qubernetes-keyonly.yaml b/config/qubernetes-keyonly.yaml new file mode 100644 index 0000000..3a949b0 --- /dev/null +++ b/config/qubernetes-keyonly.yaml @@ -0,0 +1,68 @@ +namespace: + name: quorum-test + +quorum: + # base quorum data dir as set inside each container. + Node_DataDir: /etc/quorum/qdata + # This is where all the keys are store, and/or where they are generated, as in the case of quorum-keygen. + # Either full or relative paths on the machine generating the config + Key_Dir_Base: out/config + Permissioned_Nodes_File: out/config/permissioned-nodes.json + Genesis_File: out/config/genesis.json + # related to quorum containers + quorum: + Raft_Port: 50401 + # container images at https://hub.docker.com/u/quorumengineering/ + Quorum_Version: 2.1.1 + # related to transaction manager containers + tm: + # container images at https://hub.docker.com/u/quorumengineering/ + # TODO: add support for tessera, this should hold the full container name + # in that case. + Tm_Version: 0.3.2 + Port: 9001 + +# generic geth related options +geth: + Node_WSPort: 8546 + NodeP2P_ListenAddr: 21000 + network: + # network id (1: mainnet, 3: ropsten, 4: rinkeby ... ) + id: 1101 + # public (true|false) is it a public network? + public: false + # general verbosity of geth [1..5] + verbosity: 9 + +# here you can add as many nodes as you like, name and configure them +# Note: +# 1. need to be in the list of permissioned nodes and static nodes. +# 2. keys should be set locally. +nodes: +- member: + Node_UserIdent: quorum-node01 + Key_Dir: key1 +# keep adding nodes +# - member: +# ... +- member: + Node_UserIdent: quorum-node02 + Key_Dir: key2 +- member: + Node_UserIdent: quorum-node03 + Key_Dir: key3 +- member: + Node_UserIdent: quorum-node04 + Key_Dir: key4 +- member: + Node_UserIdent: quorum-node05 + Key_Dir: key5 +- member: + Node_UserIdent: quorum-node06 + Key_Dir: key6 +- member: + Node_UserIdent: quorum-node07 + Key_Dir: key7 +- member: + Node_UserIdent: quorum-node08 + Key_Dir: key8 diff --git a/contracts/private_contract.js b/contracts/private_contract.js new file mode 100644 index 0000000..add2bc3 --- /dev/null +++ b/contracts/private_contract.js @@ -0,0 +1,22 @@ +a = eth.accounts[0] +web3.eth.defaultAccount = a; + +// abi and bytecode generated from simplestorage.sol: +// > solcjs --bin --abi simplestorage.sol +var abi = [{"constant":true,"inputs":[],"name":"storedData","outputs":[{"name":"","type":"uint256"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"x","type":"uint256"}],"name":"set","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"get","outputs":[{"name":"retVal","type":"uint256"}],"payable":false,"type":"function"},{"inputs":[{"name":"initVal","type":"uint256"}],"payable":false,"type":"constructor"}]; + +var bytecode = "0x6060604052341561000f57600080fd5b604051602080610149833981016040528080519060200190919050505b806000819055505b505b610104806100456000396000f30060606040526000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff1680632a1afcd914605157806360fe47b11460775780636d4ce63c146097575b600080fd5b3415605b57600080fd5b606160bd565b6040518082815260200191505060405180910390f35b3415608157600080fd5b6095600480803590602001909190505060c3565b005b341560a157600080fd5b60a760ce565b6040518082815260200191505060405180910390f35b60005481565b806000819055505b50565b6000805490505b905600a165627a7a72305820d5851baab720bba574474de3d09dbeaabc674a15f4dd93b974908476542c23f00029"; + +var simpleContract = web3.eth.contract(abi); +var simple = simpleContract.new(42, {from:web3.eth.accounts[0], data: bytecode, gas: 0x47b760, privateFor: ["ROAZBWtSacxXQrOe3FGAqJDyJjFePR5ce4TSIzmJ0Bc="]}, function(e, contract) { + if (e) { + console.log("err creating contract", e); + } else { + if (!contract.address) { + console.log("Contract transaction send: TransactionHash: " + contract.transactionHash + " waiting to be mined..."); + } else { + console.log("Contract mined! Address: " + contract.address); + console.log(contract); + } + } +}); diff --git a/contracts/public_contract.js b/contracts/public_contract.js new file mode 100644 index 0000000..a16cb58 --- /dev/null +++ b/contracts/public_contract.js @@ -0,0 +1,22 @@ +a = eth.accounts[0] +web3.eth.defaultAccount = a; + +// abi and bytecode generated from simplestorage.sol: +// > solcjs --bin --abi simplestorage.sol +var abi = [{"constant":true,"inputs":[],"name":"storedData","outputs":[{"name":"","type":"uint256"}],"payable":false,"type":"function"},{"constant":false,"inputs":[{"name":"x","type":"uint256"}],"name":"set","outputs":[],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"get","outputs":[{"name":"retVal","type":"uint256"}],"payable":false,"type":"function"},{"inputs":[{"name":"initVal","type":"uint256"}],"payable":false,"type":"constructor"}]; + +var bytecode = "0x6060604052341561000f57600080fd5b604051602080610149833981016040528080519060200190919050505b806000819055505b505b610104806100456000396000f30060606040526000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff1680632a1afcd914605157806360fe47b11460775780636d4ce63c146097575b600080fd5b3415605b57600080fd5b606160bd565b6040518082815260200191505060405180910390f35b3415608157600080fd5b6095600480803590602001909190505060c3565b005b341560a157600080fd5b60a760ce565b6040518082815260200191505060405180910390f35b60005481565b806000819055505b50565b6000805490505b905600a165627a7a72305820d5851baab720bba574474de3d09dbeaabc674a15f4dd93b974908476542c23f00029"; + +var simpleContract = web3.eth.contract(abi); +var simple = simpleContract.new(42, {from:web3.eth.accounts[0], data: bytecode, gas: 0x47b760}, function(e, contract) { + if (e) { + console.log("err creating contract", e); + } else { + if (!contract.address) { + console.log("Contract transaction send: TransactionHash: " + contract.transactionHash + " waiting to be mined..."); + } else { + console.log("Contract mined! Address: " + contract.address); + console.log(contract); + } + } +}); diff --git a/contracts/runscript.sh b/contracts/runscript.sh new file mode 100644 index 0000000..b64adce --- /dev/null +++ b/contracts/runscript.sh @@ -0,0 +1,2 @@ +#!/bin/ash +PRIVATE_CONFIG=$TM_HOME/tm.ipc geth --exec "loadScript(\"$1\")" attach ipc:$QUORUM_HOME/dd/geth.ipc diff --git a/delete.sh b/delete.sh new file mode 100755 index 0000000..c506559 --- /dev/null +++ b/delete.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +qctl="kubectl --namespace=quorum-test --kubeconfig=/home/libby/.go/src/github.com/ethereum/k8-quorum/k8_config --insecure-skip-tls-verify " + +$qctl delete -f out/quorum-shared-config.yaml +$qctl delete -f out/quorum-services.yaml +$qctl delete -f out/quorum-deployments.yaml +$qctl delete -f out/quorum-keyconfigs.yaml diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 0000000..848a26a --- /dev/null +++ b/deploy.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +qctl="kubectl --namespace=quorum-test --kubeconfig=/home/libby/.go/src/github.com/ethereum/k8-quorum/k8_config --insecure-skip-tls-verify " + +$qctl create -f out/quorum-shared-config.yaml +$qctl create -f out/quorum-services.yaml +$qctl create -f out/quorum-keyconfigs.yaml +$qctl create -f out/quorum-deployments.yaml diff --git a/gen-keys.sh.erb b/gen-keys.sh.erb new file mode 100644 index 0000000..35723b8 --- /dev/null +++ b/gen-keys.sh.erb @@ -0,0 +1,43 @@ +#!/bin/bash + + +<% +def set_node_template_vars(values) + @Node_Key_Dir = values["Key_Dir"] + return +end +-%> + +<% @Key_Dir_Base = @config["quorum"]["Key_Dir_Base"] %> +<% @Node_Dirs = "" %> + +<%- @nodes.each do |node| + set_node_template_vars(node.values.first) + @Node_Dirs=@Node_Dirs + "," + @Node_Key_Dir + end + @Node_Dirs[0]='' +-%> + +NODE_DIRS=<%= @Node_Dirs %> + +<%- if @Key_Dir_Base[0] == '/' %> +BASE_DIR=<%= @Key_Dir_Base %> +<%- else %> +BASE_DIR=$(pwd)/<%= @Key_Dir_Base %> +<%- end %> + +mkdir -p $BASE_DIR +IFS=', ' read -r -a array <<< "$NODE_DIRS" + +for node_key_dir in "${array[@]}"; do + pushd . + KEY_DIR=$BASE_DIR/$node_key_dir + mkdir -p $KEY_DIR + cd $KEY_DIR + echo | constellation-node --generatekeys=tm + touch password.txt + geth --keystore $KEY_DIR account new --password password.txt + bootnode -genkey nodekey.key + bootnode -nodekeyhex $(cat nodekey.key) -writeaddress > enode + popd +done diff --git a/genesis.json.erb b/genesis.json.erb new file mode 100644 index 0000000..a651215 --- /dev/null +++ b/genesis.json.erb @@ -0,0 +1,56 @@ +<% +def set_node_template_vars(values) + @Node_UserIdent = values["Node_UserIdent"] + @Node_Key_Dir = values["Key_Dir"] + return +end + +@Key_Dir_Base = @config["quorum"]["Key_Dir_Base"] +@Geth_Network_Id = @config["geth"]["network"]["id"] +@Accounts = "" +@Account_Allocs = "" +@Node_Index = 0 +-%> + +{ +"alloc": { +<%- @nodes.each_with_index do |node, indexNode| + set_node_template_vars(node.values.first) + @Keystores=Dir[@Key_Dir_Base + "/" + @Node_Key_Dir + "/UTC*"] + puts(@Accounts) + # Need to keep track of when the last account in the last node + # is being writen, so as not to have a trailing ',' in the + # genesis alloc json. + # The keystore is assumed to be the generated keystore and will + # start with UTC- and end with the account public key. + @Keystores.each_with_index do |keystore, indexKey| + #split on -- + acct="0x" + keystore.split("--")[2] + -%> "<%= acct%>": { + "balance": "1000000000000000000000000000" + }<%- if (indexNode == @nodes.size - 1) and (indexKey == @Keystores.size - 1) + else %>, <%- end %> + <%- + puts(acct) + end + -%> +<% end %> + }, + "coinbase": "0x0000000000000000000000000000000000000000", + "config": { + "byzantiumblock": 1, + "chainid": <%= @Geth_Network_Id %>, + "eip150block": 1, + "eip155block": 0, + "eip150hash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "eip158block": 1, + "isquorum":true + }, + "difficulty": "0x0", + "extradata": "0x0000000000000000000000000000000000000000000000000000000000000000", + "gaslimit": "0xe0000000", + "mixhash": "0x00000000000000000000000000000000000000647572616c65787365646c6578", + "nonce": "0x0", + "parenthash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "timestamp": "0x00" +} diff --git a/permissioned-nodes.json.erb b/permissioned-nodes.json.erb new file mode 100644 index 0000000..6785016 --- /dev/null +++ b/permissioned-nodes.json.erb @@ -0,0 +1,26 @@ +<% +def set_node_template_vars(values) + @Node_UserIdent = values["Node_UserIdent"] + @Node_Key_Dir = values["Key_Dir"] + return +end +-%> + +<% # The configured key directory of each node holds a file with the enode value. + # The enode value is obtained from that file and set in the permissioned-nodes.json +%> +<% @Key_Dir_Base = @config["quorum"]["Key_Dir_Base"] %> +[ +<%- @nodes.each_with_index do |node, indexNode| %> + <%= set_node_template_vars(node.values.first) -%> + <%- + @Enode_File = @config["quorum"]["Key_Dir_Base"] + "/" + @Node_Key_Dir + "/enode" + %> + <%- File.readlines(@Enode_File).each do |line| + @Enode = "#{line}".gsub(/\s+/, "") + end + -%> + "enode://<%= @Enode %>@%<%= "#{@Node_UserIdent}".upcase %>_SERVICE_HOST%:21000?discport=0&raftport=50401"<%- if (indexNode != @nodes.size - 1) %>,<%- end %> + +<% end -%> +] diff --git a/qubernetes b/qubernetes new file mode 100755 index 0000000..d33204e --- /dev/null +++ b/qubernetes @@ -0,0 +1,74 @@ +#!/usr/bin/env ruby + +require "yaml" +require "erb" +require 'colorize' + +# generic variables +@config = YAML.load_file("qubernetes.yaml") +@nodes = @config["nodes"] +@Raft_Port = @config["quorum"]["quorum"]["Raft_Port"] +@Node_WSPort = @config["geth"]["Node_WSPort"] +@NodeP2P_ListenAddr = @config["geth"]["NodeP2P_ListenAddr"] + +##################### +# Create config files for each node +##################### + +def set_node_template_vars(values) + @Node_UserIdent = values["Node_UserIdent"] + return +end + +# create the output directory if it doesn't exist +`mkdir -p out` + +sed_string = "" + +# make all services +# set the replacement string sed cmmand which is run on the permissioned-nodes.json, +# as the service host IPs of the nodes are not known until they are deployed, and need +# to be properly set in the permissioned-nodes.json. +#PERM_NODE_JSON=$(echo $PERM_NODE_TMPL | sed \"s/%QUORUM_DEPLOYMENT_01_SERVICE_HOST%/$QUORUM_NODE01_SERVICE_HOST/g\" | sed \"s/\\$QUORUM_DEPLOYMENT_02_SERVICE_HOST/$QUORUM_NODE02_SERVICE_HOST/g\"); +@nodes.each do |node| + set_node_template_vars(node.values.first) +# puts ("#{@Node_UserIdent}") + k8_service_host_var = ("#{@Node_UserIdent}".upcase + "_SERVICE_HOST").gsub("-", "_") + permission_node_host="#{@Node_UserIdent}".upcase + "_SERVICE_HOST" + # puts (k8_service_host_var) + sed_instruction = ' sed \"s/%' + permission_node_host + '%/$' + k8_service_host_var + '/g\"' + sed_string = sed_string + sed_instruction + " | " +end + +@Sed_Set_Node_Service_Host = sed_string[0...-2] +#puts (sed_string) + +File.open("out/quorum-shared-config.yaml", "w") do |f| + f.puts (ERB.new(File.read("quorum-shared-config.yaml.erb"), nil, "-").result) +end + +# Create the service resources +File.open("out/quorum-services.yaml", "w") do |f| + f.puts (ERB.new(File.read("quorum-services.yaml.erb"), nil, "-").result) +end + +# make all keystore resrouce (configMap, TODO: Secretes) +File.open("out/quorum-keyconfigs.yaml", "w") do |f| + f.puts (ERB.new(File.read("quorum-keystore.yaml.erb"), nil, "-").result) +end + +# make deployments +File.open("out/quorum-deployments.yaml", "w") do |f| + f.puts (ERB.new(File.read("quorum-deployment.yaml.erb"), nil, "-").result) +end + +puts("\n") +puts " Success! ".green +puts(" Kubernetes resrouce files for kubernetes have been generated in the `out/` directory.") +puts("\n") +puts(" Deploy to kubernetes:") +puts("\n") +puts(" run `/deploy.sh` with your kubernetes cluster ") +puts(" set in the script.") +puts(" or run `kubectl apply -f {Resource.yam}` on the generated files") +puts("\n") diff --git a/qubernetes.yaml b/qubernetes.yaml new file mode 120000 index 0000000..c04545a --- /dev/null +++ b/qubernetes.yaml @@ -0,0 +1 @@ +config/qubernetes-keyonly.yaml \ No newline at end of file diff --git a/quorum-config b/quorum-config new file mode 100755 index 0000000..7477e9c --- /dev/null +++ b/quorum-config @@ -0,0 +1,30 @@ +#!/usr/bin/env ruby + +require "yaml" +require "erb" + +# generic variables +@config = YAML.load_file("qubernetes.yaml") +@nodes = @config["nodes"] + +@Raft_Port = @config["quorum"]["quorum"]["Raft_Port"] +@Permissioned_Nodes_File = @config["quorum"]["Permissioned_Nodes_File"] +@Genesis_File = @config["quorum"]["Genesis_File"] +@Node_WSPort = @config["geth"]["Node_WSPort"] +@NodeP2P_ListenAddr = @config["geth"]["NodeP2P_ListenAddr"] + +##################################################### +# Generate genesis.json and permissioned-nodes.json +##################################################### + +# create genesis files with all discovered keystore accounts pre alloc with funds. +puts(@Genesis_File) +File.open(@Genesis_File, "w") do |f| + f.puts (ERB.new(File.read("genesis.json.erb"), nil, "-").result) +end + +# create permission nodes file containing all the nodes. +puts(@Permissioned_Nodes_File) +File.open(@Permissioned_Nodes_File , "w") do |f| + f.puts (ERB.new(File.read("permissioned-nodes.json.erb"), nil, "-").result) +end diff --git a/quorum-deployment.yaml.erb b/quorum-deployment.yaml.erb new file mode 100644 index 0000000..fe87c31 --- /dev/null +++ b/quorum-deployment.yaml.erb @@ -0,0 +1,231 @@ +<% +def set_node_template_vars(values) + @Node_UserIdent = values["Node_UserIdent"] + return +end +-%> +--- + +<%- @nodes.each do |node| -%> +<%= set_node_template_vars(node.values.first) -%> + +<% +@Service_Prefix = (@Node_UserIdent.upcase).gsub("-", "_") +@Geth_Verbosity = @config["geth"]["verbosity"] +@Geth_Network_Id = @config["geth"]["network"]["id"] +@Node_WSPort = @config["geth"]["Node_WSPort"] +@NodeP2P_ListenAddr = @config["geth"]["NodeP2P_ListenAddr"] + +@Node_DataDir = @config["quorum"]["Node_DataDir"] +@Quorum_Version = @config["quorum"]["quorum"]["Quorum_Version"] +@Tm_Version = @config["quorum"]["tm"]["Tm_Version"] +@Constellation_port = @config["quorum"]["tm"]["Port"] +%> + +# The quorum deployment consists of +# 1. the transaction manager / private tx container (constellation or tessera) +# 2. the quorum node container + +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: <%= @Node_UserIdent %>-deployment + namespace: <%= @config["namespace"]["name"] %> +spec: + strategy: + type: RollingUpdate + replicas: 1 + template: + metadata: + name: <%= @Node_UserIdent %>-deployment + labels: + app: qubernetes + tier: backend + name: <%= @Node_UserIdent %>-deployment + spec: + initContainers: + - name: quorum-genesis-init-container + image: quorumengineering/quorum:<%= @Quorum_Version %> + command: [ "sh" ] + args: + - "-cx" + - "if [ ! -f $QUORUM_DATA_DIR/genesis_created ]; then + /usr/local/bin/geth --datadir $QUORUM_DATA_DIR init /etc/geth/genesis/genesis-geth.json; + touch $QUORUM_DATA_DIR/genesis_created; + fi; + cp -r <%= @Node_DataDir %>/contracts-tmp <%= @Node_DataDir %>/contracts; + chmod 755 <%= @Node_DataDir %>/contracts/runscript.sh; + " + env: + - name: QUORUM_DATA_DIR + value: <%= @Node_DataDir %>/dd + volumeMounts: + - name: quorum-persistent-storage + mountPath: <%= @Node_DataDir %> + - name: genesis-config-persistent-storage + mountPath: /etc/geth/genesis/genesis-geth.json + subPath: genesis-geth.json + - name: contracts-config + mountPath: <%= @Node_DataDir%>/contracts-tmp + readOnly: false + containers: + - name: constellation + image: quorumengineering/constellation:<%= @Tm_Version %> + command: ["sh"] + args: + - "-cx" + - "chmod 600 $QUORUM_HOME/tm/tm.key; + DDIR=$QUORUM_HOME/tm; + printenv; + args=\" --url=https://$<%= @Service_Prefix %>_SERVICE_HOST:<%= @Constellation_port %>/ \ + --port=<%= @Constellation_port %> \ + --workdir=$DDIR \ + --socket=$DDIR/tm.ipc \ + --publickeys=$QUORUM_HOME/tm/tm.pub \ + --privatekeys=$QUORUM_HOME/tm/tm.key \ + --verbosity=<%= @Geth_Verbosity%> \ + --othernodes=https://$QUORUM_NODE04_SERVICE_HOST:<%= @Constellation_port %>/ \"; + /usr/local/bin/constellation-node $args 2>&1 | tee -a $QUORUM_HOME/logs/tm.log; " + ports: + - containerPort: <%= @Constellation_port %> + env: + - name: QUORUM_HOME + value: <%= @Node_DataDir %> + volumeMounts: + - name: quorum-logs-persistent-storage + mountPath: <%= @Node_DataDir %>/logs + - name: constellation-persistent-storage + mountPath: <%= @Node_DataDir %>/tm + - name: quorum-persistent-storage + mountPath: <%= @Node_DataDir %> + - name: keystore-constellation + mountPath: <%= @Node_DataDir %>/tm/tm.pub + subPath: tm.pub + - name: keystore-constellation + mountPath: <%= @Node_DataDir %>/tm/tm.key + subPath: tm.key + - name: quorum + image: quorumengineering/quorum:<%= @Quorum_Version %> + command: [ "sh" ] + # TODO: have to generate sed files + # PERM_NODE_JSON=$(echo $PERM_NODE_TMPL | sed \"s/%QUORUM-NODE01_SERVICE_HOST%/$QUORUM_NODE01_SERVICE_HOST/g\" | sed \"s/%QUORUM-NODE02_SERVICE_HOST%/$QUORUM_NODE02_SERVICE_HOST/g\"); + # sleep to give constellation some time to start up and discover the other nodes. + args: + - "-cx" + - " + sleep 5; + PERM_NODE_TMPL=$(cat $QUORUM_DATA_DIR/permissioned-nodes.json.tmpl); + PERM_NODE_JSON=$(echo $PERM_NODE_TMPL | <%= @Sed_Set_Node_Service_Host %>); + echo $PERM_NODE_JSON > $QUORUM_DATA_DIR/permissioned-nodes.json; + cp $QUORUM_DATA_DIR/permissioned-nodes.json $QUORUM_DATA_DIR/static-nodes.json; + + rm -r <%= @Node_DataDir %>/contracts-tmp; + + echo what in this dir; + ls $QUORUM_DATA_DIR; + cat /etc/geth/genesis/genesis-geth.json; + + mkdir -p /etc/geth/qdata; + chmod 644 $QUORUM_DATA_DIR/keystore/key; + touch $QUORUM_DATA_DIR/password.txt; + NETWORK_ID=<%= @Geth_Network_Id %> + raft_args=\" --raft --raftport <%= @Raft_Port %> --rpcapi admin,db,eth,debug,miner,net,shh,txpool,personal,web3,quorum,raft \"; + args=\"--rpcapi admin,db,eth,debug,miner,net,shh,txpool,personal,web3,quorum \"; + /usr/local/bin/geth \ + --datadir $QUORUM_DATA_DIR \ + $raft_args \ + --permissioned \ + --nodiscover \ + --nat=none \ + --verbosity <%= @Geth_Verbosity %> \ + --networkid $NETWORK_ID \ + --unlock 0 \ + --emitcheckpoints \ + --rpc \ + --rpcaddr 0.0.0.0 \ + --rpcport <%= @Node_WSPort %> \ + --port <%= @NodeP2P_ListenAddr %> \ + --password $QUORUM_DATA_DIR/password.txt 2>&1 | tee -a <%= @Node_DataDir%>/logs/quorum.log; " + ports: + - containerPort: <%= @Raft_Port %> + - containerPort: <%= @Node_WSPort %> + - containerPort: <%= @NodeP2P_ListenAddr %> + env: + - name: PRIVATE_CONFIG + value: <%= @Node_DataDir %>/tm/tm.ipc + - name: QUORUM_DATA_DIR + value: <%= @Node_DataDir %>/dd + - name: QUORUM_HOME + value: <%= @Node_DataDir %> + - name: TM_HOME + value: <%= @Node_DataDir %>/tm/ + volumeMounts: + - name: genesis-config-persistent-storage + mountPath: /etc/geth/genesis/genesis-geth.json + subPath: genesis-geth.json + - name: quorum-persistent-storage + mountPath: <%= @Node_DataDir%> + - name: constellation-persistent-storage + mountPath: <%= @Node_DataDir%>/tm + - name: quorum-key-config-persistent-storage + mountPath: <%= @Node_DataDir%>/dd/keystore/key + subPath: key + - name: quorum-logs-persistent-storage + mountPath: <%= @Node_DataDir %>/logs + - name: quorum-nodekey + mountPath: <%= @Node_DataDir%>/dd/geth/nodekey + subPath: nodekey + - name: quorum-permissioned-config + mountPath: <%= @Node_DataDir%>/dd/permissioned-nodes.json.tmpl + subPath: permissioned-nodes.json + - name: quorum-permissioned-config + mountPath: <%= @Node_DataDir%>/dd/static-nodes.json.tmpl + subPath: permissioned-nodes.json + volumes: + - name: quorum-permissioned-config + configMap: + name: quorum-permissioned-config + items: + - key: permissioned-nodes.json + path: permissioned-nodes.json + - name: genesis-config-persistent-storage + configMap: + name: genesis-config + items: + - key: genesis-geth.json + path: genesis-geth.json + - name: contracts-config + configMap: + name: contracts-config + - name: keystore-constellation + configMap: + name: <%= @Node_UserIdent %>-constellation-key-config + items: + - key: tm.pub + path: tm.pub + - key: tm.key + path: tm.key + - name: quorum-key-config-persistent-storage + configMap: + name: <%= @Node_UserIdent %>-account-key-config + items: + - key: key + path: key + - name: quorum-nodekey + configMap: + name: <%= @Node_UserIdent %>-nodekey-config + items: + - key: nodekey + path: nodekey + - name: quorum-persistent-storage + hostPath: + path: /var/lib/docker/geth-storage/<%= @Node_UserIdent %> + - name: constellation-persistent-storage + hostPath: + path: /var/lib/docker/geth-storage/tm-<%= @Node_UserIdent %> + - name: quorum-logs-persistent-storage + hostPath: + path: /var/lib/docker/geth-storage/<%= @Node_UserIdent %>-logs + +<% end -%> diff --git a/quorum-init b/quorum-init new file mode 100755 index 0000000..48ff8a6 --- /dev/null +++ b/quorum-init @@ -0,0 +1,15 @@ +#!/bin/bash + +# create the output directory if +# it doesn't exist +mkdir -p out + +# generate the keygen script +# to generate nodekey, account keys, and +# quorum transaction manger (tm.pub,tm.key) keys. +./quorum-keygen + +# generate the appropriate config files: +# permissioned-nodes.json +# genesis.json +./quorum-config diff --git a/quorum-keygen b/quorum-keygen new file mode 100755 index 0000000..d176e89 --- /dev/null +++ b/quorum-keygen @@ -0,0 +1,18 @@ +#!/usr/bin/env ruby + +require "yaml" +require "erb" + +# generic variables +@config = YAML.load_file("qubernetes.yaml") +@nodes = @config["nodes"] +#@Raft_Port = @config["quorum"]["quorum"]["Raft_Port"] +#@Node_WSPort = @config["geth"]["Node_WSPort"] +#@NodeP2P_ListenAddr = @config["geth"]["NodeP2P_ListenAddr"] + +File.open("gen-node-keys.sh", "w") do |f| + f.puts (ERB.new(File.read("gen-keys.sh.erb"), nil, "-").result) +end + +`chmod 755 gen-node-keys.sh` +`./gen-node-keys.sh` diff --git a/quorum-keystore.yaml.erb b/quorum-keystore.yaml.erb new file mode 100644 index 0000000..4617872 --- /dev/null +++ b/quorum-keystore.yaml.erb @@ -0,0 +1,84 @@ +<% +# Create Kubernetes resources for quorum keys: +# nodekey (geth) +# keystore account key (geth) +# transaction manager key `tm` (quorum: constellation | tessera) +def set_node_template_vars(values) + @Node_UserIdent = values["Node_UserIdent"] + @Node_Key_Dir = values["Key_Dir"] +# @Constellation_key = values["tm_key"]["name"] +# @Nodekey = values["nodekey"]["name"] +# @Keystore_name = values["keystore"]["name"] + return +end + +# keys are named the same inside their respective directories. +@Keystore_Dir = @config["quorum"]["Key_Dir_Base"] +-%> + + +<%- @nodes.each do |node| -%> +<%= set_node_template_vars(node.values.first) -%> + +# kubectl create configmap game-config --from-file=configure-pod-container/dd1/key +# the key used for private transactions +# quorum transaction manager keys +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: <%= @Node_UserIdent %>-constellation-key-config + namespace: <%= @config["namespace"]["name"] %> + labels: + app: qubernetes + name: <%= @Node_UserIdent %>-constellation-key-config +data: + tm.key: |- +<%- File.readlines("#{@Keystore_Dir}/#{@Node_Key_Dir}/tm.key").each do |line| -%> + <%= line -%> +<% end -%> + + tm.pub: |- +<%- File.readlines("#{@Keystore_Dir}/#{@Node_Key_Dir}/tm.pub").each do |line| -%> + <%= line -%> +<% end -%> + +--- +# kubectl create configmap game-config --from-file=configure-pod-container/dd1/key +# nodekey (enode) (geth/ethereum) +apiVersion: v1 +kind: ConfigMap +metadata: + name: <%= @Node_UserIdent %>-nodekey-config + namespace: <%= @config["namespace"]["name"] %> + labels: + app: qubernetes + name: <%= @Node_UserIdent %>-nodekey-config +data: + nodekey: | +<%- File.readlines("#{@Keystore_Dir}/#{@Node_Key_Dir}/nodekey.key").each do |line| -%> + <%= line -%> +<% end -%> + + +# ethereum / geth account keys (keystore) +--- +# kubectl create configmap game-config --from-file=configure-pod-container/dd1/key +apiVersion: v1 +kind: ConfigMap +metadata: + name: <%= @Node_UserIdent %>-account-key-config + namespace: <%= @config["namespace"]["name"] %> + labels: + app: qubernetes + name: <%= @Node_UserIdent %>-account-key-config +data: + key: |- + +<%- @Keystore_File=Dir[@Keystore_Dir + "/" + @Node_Key_Dir + "/UTC*"][0] %> +<%- File.readlines("#{@Keystore_File}").each do |line| -%> + <%= line -%> +<% end -%> + +# end node +<% end %> diff --git a/quorum-services.yaml.erb b/quorum-services.yaml.erb new file mode 100644 index 0000000..75211e2 --- /dev/null +++ b/quorum-services.yaml.erb @@ -0,0 +1,52 @@ +<% +def set_node_template_vars(values) + @Node_UserIdent = values["Node_UserIdent"] + return +end +-%> + +<%- @nodes.each do |node| -%> + <%= set_node_template_vars(node.values.first) -%> + +<% + @Service_Prefix = (@Node_UserIdent.upcase).gsub("-", "_") + @TM_Port = @config["quorum"]["tm"]["Port"] + @Node_WSPort = @config["geth"]["Node_WSPort"] + @NodeP2P_ListenAddr = @config["geth"]["NodeP2P_ListenAddr"] +%> + +--- +apiVersion: v1 +kind: Service +metadata: + namespace: <%= @config["namespace"]["name"] %> + name: <%= @Node_UserIdent %> + labels: + app: qubernetes + tier: backend + name: <%= @Node_UserIdent %> +spec: + selector: + app: qubernetes + tier: backend + name: <%= @Node_UserIdent %>-deployment + type: ClusterIP + ports: + - name: constellation + protocol: TCP + port: <%= @TM_Port %> + targetPort: <%= @TM_Port %> + # default 8545 + - name: wsrpc-listener + protocol: TCP + port: <%= @Node_WSPort %> + targetPort: <%= @Node_WSPort %> + - name: quorum-listener + protocol: TCP + port: <%= @NodeP2P_ListenAddr %> + targetPort: <%= @NodeP2P_ListenAddr %> + - name: raft + protocol: TCP + targetPort: <%= @Raft_Port %> + port: <%= @Raft_Port %> +<% end -%> diff --git a/quorum-shared-config.yaml.erb b/quorum-shared-config.yaml.erb new file mode 100644 index 0000000..793491f --- /dev/null +++ b/quorum-shared-config.yaml.erb @@ -0,0 +1,66 @@ +<%- + @Permissioned_Nodes_File = @config["quorum"]["Permissioned_Nodes_File"] + @Genesis_File = @config["quorum"]["Genesis_File"] +%> + +apiVersion: v1 +kind: Namespace +metadata: + name: <%= @config["namespace"]["name"] %> + labels: + name: <%= @config["namespace"]["name"] %> + +--- +# kubectl create configmap game-config --from-file=configure-pod-container/dd1/key +apiVersion: v1 +kind: ConfigMap +metadata: + name: quorum-permissioned-config + namespace: <%= @config["namespace"]["name"] %> + labels: + app: qubernetes + name: quorum-permissioned-config +data: + permissioned-nodes.json: | + +<%- File.readlines(@Permissioned_Nodes_File).each do |line| -%> + <%= line -%> +<% end -%> + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: genesis-config + namespace: <%= @config["namespace"]["name"] %> + labels: + app: qubernetes + name: genesis-config +data: + genesis-geth.json: |- +<%- File.readlines(@Genesis_File).each do |line| -%> + <%= line -%> +<% end -%> + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: contracts-config + namespace: <%= @config["namespace"]["name"] %> + labels: + app: qubernetes + name: contracts-config +data: + runscript.sh: |- +<%- File.readlines("contracts/runscript.sh").each do |line| -%> + <%= line -%> +<% end -%> + private_contract.js: |- +<%- File.readlines("contracts/private_contract.js").each do |line| -%> + <%= line -%> +<% end -%> + public_contract.js: |- +<%- File.readlines("contracts/public_contract.js").each do |line| -%> + <%= line -%> +<% end -%>