From ef27e65c8d7c5d6345becc5dd838664e7d7e55cc Mon Sep 17 00:00:00 2001 From: Jim Zhang Date: Thu, 14 Sep 2017 15:01:51 -0400 Subject: [PATCH] FAB-6128 use docker upload for TLS materials pass the client TLS keyCert pair to the shim container, by uploading them as files client.key and client.crt in /etc/hyperledger/fabric during startup of the chaincode container Note that this change applies to all chaincode handling, including golang and java In addition, the golang shim has been modified to find the TLS client key and cert from the files mentioned above Change-Id: Iedae111dfad4d10a554946abb92d81e478ac6891 Signed-off-by: Jim Zhang --- core/chaincode/chaincode_support.go | 39 ++++++++++++----- core/chaincode/chaincode_support_test.go | 42 ++++++++++++++----- core/chaincode/shim/chaincode.go | 19 ++++++++- core/container/api/core.go | 2 +- core/container/controller.go | 3 +- .../dockercontroller/dockercontroller.go | 38 ++++++++++++++++- .../dockercontroller/dockercontroller_test.go | 42 +++++++++++++------ .../inproccontroller/inproccontroller.go | 2 +- 8 files changed, 147 insertions(+), 40 deletions(-) diff --git a/core/chaincode/chaincode_support.go b/core/chaincode/chaincode_support.go index 8dd07b0b549..f0f90260cd5 100644 --- a/core/chaincode/chaincode_support.go +++ b/core/chaincode/chaincode_support.go @@ -20,6 +20,7 @@ import ( "fmt" "io" "path/filepath" + "reflect" "strconv" "strings" "sync" @@ -56,6 +57,10 @@ const ( //HistoryQueryExecutorKey is used to attach ledger history query executor context HistoryQueryExecutorKey key = "historyqueryexecutorkey" + + // Mutual TLS auth client key and cert paths in the chaincode container + TLSClientKeyPath string = "/etc/hyperledger/fabric/client.key" + TLSClientCertPath string = "/etc/hyperledger/fabric/client.crt" ) //this is basically the singleton that supports the @@ -356,15 +361,20 @@ func (chaincodeSupport *ChaincodeSupport) sendReady(context context.Context, ccc return err } -func (chaincodeSupport *ChaincodeSupport) appendTLScerts(args []string, keyPair *accesscontrol.CertAndPrivKeyPair) []string { +// returns a map of file path <-> []byte for all files related to TLS +func (chaincodeSupport *ChaincodeSupport) getTLSFiles(keyPair *accesscontrol.CertAndPrivKeyPair) map[string][]byte { if keyPair == nil { - return args + return nil + } + + return map[string][]byte{ + TLSClientKeyPath: []byte(keyPair.Key), + TLSClientCertPath: []byte(keyPair.Cert), } - return append(args, []string{"--key", keyPair.Key, "--cert", keyPair.Cert}...) } //get args and env given chaincodeID -func (chaincodeSupport *ChaincodeSupport) getArgsAndEnv(cccid *ccprovider.CCContext, cLang pb.ChaincodeSpec_Type) (args []string, envs []string, err error) { +func (chaincodeSupport *ChaincodeSupport) getLaunchConfigs(cccid *ccprovider.CCContext, cLang pb.ChaincodeSpec_Type) (args []string, envs []string, filesToUpload map[string][]byte, err error) { canName := cccid.GetCanonicalName() envs = []string{"CORE_CHAINCODE_ID_NAME=" + canName} @@ -382,12 +392,14 @@ func (chaincodeSupport *ChaincodeSupport) getArgsAndEnv(cccid *ccprovider.CCCont if chaincodeSupport.peerTLS { certKeyPair, err = chaincodeSupport.auth.Generate(cccid.GetCanonicalName()) if err != nil { - return nil, nil, errors.WithMessage(err, fmt.Sprintf("failed generating TLS cert for %s", cccid.GetCanonicalName())) + return nil, nil, nil, errors.WithMessage(err, fmt.Sprintf("failed generating TLS cert for %s", cccid.GetCanonicalName())) } envs = append(envs, "CORE_PEER_TLS_ENABLED=true") if chaincodeSupport.peerTLSSvrHostOrd != "" { envs = append(envs, "CORE_PEER_TLS_SERVERHOSTOVERRIDE="+chaincodeSupport.peerTLSSvrHostOrd) } + envs = append(envs, fmt.Sprintf("CORE_TLS_CLIENT_KEY_PATH=%s", TLSClientKeyPath)) + envs = append(envs, fmt.Sprintf("CORE_TLS_CLIENT_CERT_PATH=%s", TLSClientCertPath)) } else { envs = append(envs, "CORE_PEER_TLS_ENABLED=false") } @@ -406,17 +418,23 @@ func (chaincodeSupport *ChaincodeSupport) getArgsAndEnv(cccid *ccprovider.CCCont switch cLang { case pb.ChaincodeSpec_GOLANG, pb.ChaincodeSpec_CAR: args = []string{"chaincode", fmt.Sprintf("-peer.address=%s", chaincodeSupport.peerAddress)} - args = theChaincodeSupport.appendTLScerts(args, certKeyPair) case pb.ChaincodeSpec_JAVA: args = []string{"java", "-jar", "chaincode.jar", "--peerAddress", chaincodeSupport.peerAddress} case pb.ChaincodeSpec_NODE: args = []string{"/bin/sh", "-c", fmt.Sprintf("cd /usr/local/src; node chaincode.js --peer.address %s", chaincodeSupport.peerAddress)} + default: - return nil, nil, errors.Errorf("unknown chaincodeType: %s", cLang) + return nil, nil, nil, errors.Errorf("unknown chaincodeType: %s", cLang) } + + filesToUpload = theChaincodeSupport.getTLSFiles(certKeyPair) + chaincodeLogger.Debugf("Executable is %s", args[0]) chaincodeLogger.Debugf("Args %v", args) - return args, envs, nil + chaincodeLogger.Debugf("Envs %v", envs) + chaincodeLogger.Debugf("FilesToUpload %v", reflect.ValueOf(filesToUpload).MapKeys()) + + return args, envs, filesToUpload, nil } //launchAndWaitForRegister will launch container if not already running. Use @@ -467,7 +485,7 @@ func (chaincodeSupport *ChaincodeSupport) launchAndWaitForRegister(ctxt context. //launch the chaincode - args, env, err := chaincodeSupport.getArgsAndEnv(cccid, cLang) + args, env, filesToUpload, err := chaincodeSupport.getLaunchConfigs(cccid, cLang) if err != nil { return err } @@ -487,7 +505,8 @@ func (chaincodeSupport *ChaincodeSupport) launchAndWaitForRegister(ctxt context. return nil } - sir := container.StartImageReq{CCID: ccintf.CCID{ChaincodeSpec: cds.ChaincodeSpec, NetworkID: chaincodeSupport.peerNetworkID, PeerID: chaincodeSupport.peerID, Version: cccid.Version}, Builder: builder, Args: args, Env: env, PrelaunchFunc: preLaunchFunc} + ccid := ccintf.CCID{ChaincodeSpec: cds.ChaincodeSpec, NetworkID: chaincodeSupport.peerNetworkID, PeerID: chaincodeSupport.peerID, Version: cccid.Version} + sir := container.StartImageReq{CCID: ccid, Builder: builder, Args: args, Env: env, FilesToUpload: filesToUpload, PrelaunchFunc: preLaunchFunc} ipcCtxt := context.WithValue(ctxt, ccintf.GetCCHandlerKey(), chaincodeSupport) diff --git a/core/chaincode/chaincode_support_test.go b/core/chaincode/chaincode_support_test.go index 4fd8016b964..b5ca209a9b6 100644 --- a/core/chaincode/chaincode_support_test.go +++ b/core/chaincode/chaincode_support_test.go @@ -639,33 +639,53 @@ func getHistory(t *testing.T, chainID, ccname string, ccSide *mockpeer.MockCCCom return nil } -func getArgsAndEnv(t *testing.T, auth accesscontrol.Authenticator) { +func getLaunchConfigs(t *testing.T, auth accesscontrol.Authenticator) { newCCSupport := &ChaincodeSupport{peerTLS: true, chaincodeLogLevel: "debug", shimLogLevel: "info"} //set the authenticator for generating TLS stuff newCCSupport.auth = auth ccContext := ccprovider.NewCCContext("dummyChannelId", "mycc", "v0", "dummyTxid", false, nil, nil) - args, envs, err := newCCSupport.getArgsAndEnv(ccContext, pb.ChaincodeSpec_NODE) + args, envs, filesToUpload, err := newCCSupport.getLaunchConfigs(ccContext, pb.ChaincodeSpec_GOLANG) if err != nil { - t.Fatalf("calling getArgsAndEnv() failed with error %s", err) + t.Fatalf("calling getLaunchConfigs() failed with error %s", err) } + if len(args) != 2 { + t.Fatalf("calling getLaunchConfigs() for golang chaincode should have returned an array of 2 elements for Args, but got %v", args) + } + if args[0] != "chaincode" || !strings.HasPrefix(args[1], "-peer.address") { + t.Fatalf("calling getLaunchConfigs() should have returned the start command for golang chaincode, but got %v", args) + } + if len(envs) != 6 { + t.Fatalf("calling getLaunchConfigs() with TLS enabled should have returned an array of 6 elements for Envs, but got %v", envs) + } + if envs[0] != "CORE_CHAINCODE_ID_NAME=mycc:v0" || envs[1] != "CORE_PEER_TLS_ENABLED=true" || + envs[2] != "CORE_TLS_CLIENT_KEY_PATH=/etc/hyperledger/fabric/client.key" || envs[3] != "CORE_TLS_CLIENT_CERT_PATH=/etc/hyperledger/fabric/client.crt" || + envs[4] != "CORE_CHAINCODE_LOGGING_LEVEL=debug" || envs[5] != "CORE_CHAINCODE_LOGGING_SHIM=info" { + t.Fatalf("calling getLaunchConfigs() with TLS enabled should have returned the proper environment variables, but got %v", envs) + } + if len(filesToUpload) != 2 { + t.Fatalf("calling getLaunchConfigs() with TLS enabled should have returned an array of 2 elements for filesToUpload, but got %v", len(filesToUpload)) + } + + args, envs, _, err = newCCSupport.getLaunchConfigs(ccContext, pb.ChaincodeSpec_NODE) if len(args) != 3 { - t.Fatalf("calling getArgsAndEnv() should have returned an array of 3 elements for Args, but got %v", args) + t.Fatalf("calling getLaunchConfigs() for node chaincode should have returned an array of 3 elements for Args, but got %v", args) } - if args[0] != "/bin/sh" || args[1] != "-c" && !strings.HasPrefix(args[2], "cd /usr/local/src; node chaincode.js --peer.address") { - t.Fatalf("calling getArgsAndEnv() should have returned the start command for node.js chaincode, but got %v", args) + if args[0] != "/bin/sh" || args[1] != "-c" || !strings.HasPrefix(args[2], "cd /usr/local/src; node chaincode.js --peer.address") { + t.Fatalf("calling getLaunchConfigs() should have returned the start command for node.js chaincode, but got %v", args) } + newCCSupport.peerTLS = false + args, envs, _, err = newCCSupport.getLaunchConfigs(ccContext, pb.ChaincodeSpec_GOLANG) if len(envs) != 4 { - t.Fatalf("calling getArgsAndEnv() should have returned an array of 4 elements for Envs, but got %v", envs) + t.Fatalf("calling getLaunchConfigs() with TLS disabled should have returned an array of 4 elements for Envs, but got %v", envs) } - - if envs[0] != "CORE_CHAINCODE_ID_NAME=mycc:v0" || envs[1] != "CORE_PEER_TLS_ENABLED=true" || + if envs[0] != "CORE_CHAINCODE_ID_NAME=mycc:v0" || envs[1] != "CORE_PEER_TLS_ENABLED=false" || envs[2] != "CORE_CHAINCODE_LOGGING_LEVEL=debug" || envs[3] != "CORE_CHAINCODE_LOGGING_SHIM=info" { - t.Fatalf("calling getArgsAndEnv() should have returned the proper environment variables, but got %v", envs) + t.Fatalf("calling getLaunchConfigs() with TLS disabled should have returned the proper environment variables, but got %v", envs) } } @@ -716,7 +736,7 @@ func TestCCFramework(t *testing.T) { getHistory(t, chainID, ccname, ccSide) //just use the previous authhandler for generating TLS key/pair - getArgsAndEnv(t, theChaincodeSupport.auth) + getLaunchConfigs(t, theChaincodeSupport.auth) ccSide.Quit() } diff --git a/core/chaincode/shim/chaincode.go b/core/chaincode/shim/chaincode.go index 1c639c1cdd5..1a82cb7dbec 100644 --- a/core/chaincode/shim/chaincode.go +++ b/core/chaincode/shim/chaincode.go @@ -22,6 +22,7 @@ import ( "flag" "fmt" "io" + "io/ioutil" "os" "strings" "unicode/utf8" @@ -88,8 +89,22 @@ var streamGetter peerStreamGetter func userChaincodeStreamGetter(name string) (PeerChaincodeStream, error) { flag.StringVar(&peerAddress, "peer.address", "", "peer address") if comm.TLSEnabled() { - flag.StringVar(&key, "key", "", "key in BASE64") - flag.StringVar(&cert, "cert", "", "certificate in BASE64") + keyPath := viper.GetString("tls.client.key.path") + certPath := viper.GetString("tls.client.cert.path") + + data, err1 := ioutil.ReadFile(keyPath) + if err1 != nil { + chaincodeLogger.Errorf("Error trying to read file content %s: %s", keyPath, err1) + return nil, fmt.Errorf("Error trying to read file content %s: %s", keyPath, err1) + } + key = string(data) + + data, err1 = ioutil.ReadFile(certPath) + if err1 != nil { + chaincodeLogger.Errorf("Error trying to read file content %s: %s", certPath, err1) + return nil, fmt.Errorf("Error trying to read file content %s: %s", certPath, err1) + } + cert = string(data) } flag.Parse() diff --git a/core/container/api/core.go b/core/container/api/core.go index 352dca38e53..c30f90d5a58 100644 --- a/core/container/api/core.go +++ b/core/container/api/core.go @@ -30,7 +30,7 @@ type PrelaunchFunc func() error //VM is an abstract virtual image for supporting arbitrary virual machines type VM interface { Deploy(ctxt context.Context, ccid ccintf.CCID, args []string, env []string, reader io.Reader) error - Start(ctxt context.Context, ccid ccintf.CCID, args []string, env []string, builder BuildSpecFactory, preLaunchFunc PrelaunchFunc) error + Start(ctxt context.Context, ccid ccintf.CCID, args []string, env []string, filesToUpload map[string][]byte, builder BuildSpecFactory, preLaunchFunc PrelaunchFunc) error Stop(ctxt context.Context, ccid ccintf.CCID, timeout uint, dontkill bool, dontremove bool) error Destroy(ctxt context.Context, ccid ccintf.CCID, force bool, noprune bool) error GetVMName(ccID ccintf.CCID, format func(string) (string, error)) (string, error) diff --git a/core/container/controller.go b/core/container/controller.go index ab3dc27b981..6770042175b 100644 --- a/core/container/controller.go +++ b/core/container/controller.go @@ -156,13 +156,14 @@ type StartImageReq struct { Builder api.BuildSpecFactory Args []string Env []string + FilesToUpload map[string][]byte PrelaunchFunc api.PrelaunchFunc } func (si StartImageReq) do(ctxt context.Context, v api.VM) VMCResp { var resp VMCResp - if err := v.Start(ctxt, si.CCID, si.Args, si.Env, si.Builder, si.PrelaunchFunc); err != nil { + if err := v.Start(ctxt, si.CCID, si.Args, si.Env, si.FilesToUpload, si.Builder, si.PrelaunchFunc); err != nil { resp = VMCResp{Err: err} } else { resp = VMCResp{} diff --git a/core/container/dockercontroller/dockercontroller.go b/core/container/dockercontroller/dockercontroller.go index 2394b7e223b..147b899d732 100644 --- a/core/container/dockercontroller/dockercontroller.go +++ b/core/container/dockercontroller/dockercontroller.go @@ -17,7 +17,9 @@ limitations under the License. package dockercontroller import ( + "archive/tar" "bytes" + "compress/gzip" "encoding/hex" "fmt" "io" @@ -59,6 +61,9 @@ type DockerVM struct { type dockerClient interface { // CreateContainer creates a docker container, returns an error in case of failure CreateContainer(opts docker.CreateContainerOptions) (*docker.Container, error) + // UploadToContainer uploads a tar archive to be extracted to a path in the + // filesystem of the container. + UploadToContainer(id string, opts docker.UploadToContainerOptions) error // StartContainer starts a docker container, returns an error in case of failure StartContainer(id string, cfg *docker.HostConfig) error // AttachToContainer attaches to a docker container, returns an error in case of @@ -211,7 +216,7 @@ func (vm *DockerVM) Deploy(ctxt context.Context, ccid ccintf.CCID, //Start starts a container using a previously created docker image func (vm *DockerVM) Start(ctxt context.Context, ccid ccintf.CCID, - args []string, env []string, builder container.BuildSpecFactory, prelaunchFunc container.PrelaunchFunc) error { + args []string, env []string, filesToUpload map[string][]byte, builder container.BuildSpecFactory, prelaunchFunc container.PrelaunchFunc) error { imageID, err := vm.GetVMName(ccid, formatImageName) if err != nil { return err @@ -338,6 +343,37 @@ func (vm *DockerVM) Start(ctxt context.Context, ccid ccintf.CCID, }() } + // upload specified files to the container before starting it + // this can be used for configurations such as TLS key and certs + if len(filesToUpload) != 0 { + // the docker upload API takes a tar file, so we need to first + // consolidate the file entries to a tar + payload := bytes.NewBuffer(nil) + gw := gzip.NewWriter(payload) + tw := tar.NewWriter(gw) + + for path, fileToUpload := range filesToUpload { + cutil.WriteBytesToPackage(path, fileToUpload, tw) + } + + // Write the tar file out + if err = tw.Close(); err != nil { + return fmt.Errorf("Error writing files to upload to Docker instance into a temporary tar blob: %s", err) + } + + gw.Close() + + err = client.UploadToContainer(containerID, docker.UploadToContainerOptions{ + InputStream: bytes.NewReader(payload.Bytes()), + Path: "/", + NoOverwriteDirNonDir: false, + }) + + if err != nil { + return fmt.Errorf("Error uploading files to the container instance %s: %s", containerID, err) + } + } + if prelaunchFunc != nil { if err = prelaunchFunc(); err != nil { return err diff --git a/core/container/dockercontroller/dockercontroller_test.go b/core/container/dockercontroller/dockercontroller_test.go index 9aca26c3c04..614e70ac8c5 100644 --- a/core/container/dockercontroller/dockercontroller_test.go +++ b/core/container/dockercontroller/dockercontroller_test.go @@ -101,25 +101,34 @@ func Test_Start(t *testing.T) { ccid := ccintf.CCID{ChaincodeSpec: &pb.ChaincodeSpec{ChaincodeId: &pb.ChaincodeID{Name: "simple"}}} args := make([]string, 1) env := make([]string, 1) + files := map[string][]byte{ + "hello": []byte("world"), + } ctx := context.Background() // Failure cases // case 1: getMockClient returns error dvm.getClientFnc = getMockClient getClientErr = true - err := dvm.Start(ctx, ccid, args, env, nil, nil) + err := dvm.Start(ctx, ccid, args, env, files, nil, nil) testerr(t, err, false) getClientErr = false // case 2: dockerClient.CreateContainer returns error createErr = true - err = dvm.Start(ctx, ccid, args, env, nil, nil) + err = dvm.Start(ctx, ccid, args, env, files, nil, nil) testerr(t, err, false) createErr = false - // case 3: dockerClient.CreateContainer returns docker.noSuchImgErr + // case 3: dockerClient.UploadToContainer returns error + uploadErr = true + err = dvm.Start(ctx, ccid, args, env, files, nil, nil) + testerr(t, err, false) + uploadErr = false + + // case 4: dockerClient.StartContainer returns docker.noSuchImgErr noSuchImgErr = true - err = dvm.Start(ctx, ccid, args, env, nil, nil) + err = dvm.Start(ctx, ccid, args, env, files, nil, nil) testerr(t, err, false) chaincodePath := "github.com/hyperledger/fabric/examples/chaincode/go/chaincode_example01" @@ -137,34 +146,34 @@ func Test_Start(t *testing.T) { // docker.noSuchImgErr and dockerClient.Start returns error viper.Set("vm.docker.attachStdout", true) startErr = true - err = dvm.Start(ctx, ccid, args, env, bldr, nil) + err = dvm.Start(ctx, ccid, args, env, files, bldr, nil) testerr(t, err, false) startErr = false // Success cases - err = dvm.Start(ctx, ccid, args, env, bldr, nil) + err = dvm.Start(ctx, ccid, args, env, files, bldr, nil) testerr(t, err, true) noSuchImgErr = false // dockerClient.StopContainer returns error stopErr = true - err = dvm.Start(ctx, ccid, args, env, nil, nil) + err = dvm.Start(ctx, ccid, args, env, files, nil, nil) testerr(t, err, true) stopErr = false // dockerClient.KillContainer returns error killErr = true - err = dvm.Start(ctx, ccid, args, env, nil, nil) + err = dvm.Start(ctx, ccid, args, env, files, nil, nil) testerr(t, err, true) killErr = false // dockerClient.RemoveContainer returns error removeErr = true - err = dvm.Start(ctx, ccid, args, env, nil, nil) + err = dvm.Start(ctx, ccid, args, env, files, nil, nil) testerr(t, err, true) removeErr = false - err = dvm.Start(ctx, ccid, args, env, nil, nil) + err = dvm.Start(ctx, ccid, args, env, files, nil, nil) testerr(t, err, true) //test preLaunchFunc works correctly @@ -174,7 +183,7 @@ func Test_Start(t *testing.T) { return nil } - err = dvm.Start(ctx, ccid, args, env, nil, preLaunchFunc) + err = dvm.Start(ctx, ccid, args, env, files, nil, preLaunchFunc) testerr(t, err, true) assert.Equal(t, preLaunchStr, "set") @@ -182,7 +191,7 @@ func Test_Start(t *testing.T) { return fmt.Errorf("testing error path") } - err = dvm.Start(ctx, ccid, args, env, nil, preLaunchFunc) + err = dvm.Start(ctx, ccid, args, env, files, nil, preLaunchFunc) testerr(t, err, false) } @@ -296,7 +305,7 @@ type mockClient struct { noSuchImgErrReturned bool } -var getClientErr, createErr, noSuchImgErr, buildErr, removeImgErr, +var getClientErr, createErr, uploadErr, noSuchImgErr, buildErr, removeImgErr, startErr, stopErr, killErr, removeErr bool func (c *mockClient) CreateContainer(options docker.CreateContainerOptions) (*docker.Container, error) { @@ -316,6 +325,13 @@ func (c *mockClient) StartContainer(id string, cfg *docker.HostConfig) error { return nil } +func (c *mockClient) UploadToContainer(id string, opts docker.UploadToContainerOptions) error { + if uploadErr { + return errors.New("Error uploading archive to the container") + } + return nil +} + func (c *mockClient) AttachToContainer(opts docker.AttachToContainerOptions) error { if opts.Success != nil { opts.Success <- struct{}{} diff --git a/core/container/inproccontroller/inproccontroller.go b/core/container/inproccontroller/inproccontroller.go index ddd391b66b9..3513998b734 100644 --- a/core/container/inproccontroller/inproccontroller.go +++ b/core/container/inproccontroller/inproccontroller.go @@ -154,7 +154,7 @@ func (ipc *inprocContainer) launchInProc(ctxt context.Context, id string, args [ } //Start starts a previously registered system codechain -func (vm *InprocVM) Start(ctxt context.Context, ccid ccintf.CCID, args []string, env []string, builder container.BuildSpecFactory, prelaunchFunc container.PrelaunchFunc) error { +func (vm *InprocVM) Start(ctxt context.Context, ccid ccintf.CCID, args []string, env []string, filesToUpload map[string][]byte, builder container.BuildSpecFactory, prelaunchFunc container.PrelaunchFunc) error { path := ccid.ChaincodeSpec.ChaincodeId.Path ipctemplate := typeRegistry[path]