From 83b09a154928d1f6149af89068962e3efe295af8 Mon Sep 17 00:00:00 2001 From: DuodenumL Date: Tue, 29 Mar 2022 18:51:40 +0800 Subject: [PATCH] reuse http client (#570) --- engine/docker/docker.go | 45 +++--------- engine/docker/helper.go | 20 +----- engine/factory/factory.go | 5 -- engine/virt/virt.go | 4 +- go.mod | 3 +- go.sum | 6 ++ utils/http.go | 140 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 160 insertions(+), 63 deletions(-) create mode 100644 utils/http.go diff --git a/engine/docker/docker.go b/engine/docker/docker.go index 6cb1066f6..bd00c45e7 100644 --- a/engine/docker/docker.go +++ b/engine/docker/docker.go @@ -2,18 +2,16 @@ package docker import ( "context" - "fmt" - "io/ioutil" "net/http" - "os" + "strings" "github.com/projecteru2/core/engine" enginetypes "github.com/projecteru2/core/engine/types" "github.com/projecteru2/core/log" coretypes "github.com/projecteru2/core/types" + "github.com/projecteru2/core/utils" dockerapi "github.com/docker/docker/client" - "github.com/docker/go-connections/tlsconfig" ) const ( @@ -32,44 +30,19 @@ type Engine struct { // MakeClient make docker cli func MakeClient(ctx context.Context, config coretypes.Config, nodename, endpoint, ca, cert, key string) (engine.API, error) { var client *http.Client - if config.CertPath != "" && ca != "" && cert != "" && key != "" { // nolint - caFile, err := ioutil.TempFile(config.CertPath, fmt.Sprintf("ca-%s", nodename)) + var err error + if strings.HasPrefix(endpoint, "unix://") { + client = utils.GetUnixSockClient() + } else { + client, err = utils.GetHTTPSClient(ctx, config.CertPath, nodename, ca, cert, key) if err != nil { + log.Errorf(ctx, "[MakeClient] GetHTTPSClient for %s %s error: %v", nodename, endpoint, err) return nil, err } - certFile, err := ioutil.TempFile(config.CertPath, fmt.Sprintf("cert-%s", nodename)) - if err != nil { - return nil, err - } - keyFile, err := ioutil.TempFile(config.CertPath, fmt.Sprintf("key-%s", nodename)) - if err != nil { - return nil, err - } - if err = dumpFromString(ctx, caFile, certFile, keyFile, ca, cert, key); err != nil { - return nil, err - } - options := tlsconfig.Options{ - CAFile: caFile.Name(), - CertFile: certFile.Name(), - KeyFile: keyFile.Name(), - InsecureSkipVerify: true, - } - defer os.Remove(caFile.Name()) - defer os.Remove(certFile.Name()) - defer os.Remove(keyFile.Name()) - tlsc, err := tlsconfig.Client(options) - if err != nil { - return nil, err - } - client = &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: tlsc, - }, - } } log.Debugf(ctx, "[MakeDockerEngine] Create new http.Client for %s, %s", endpoint, config.Docker.APIVersion) - return makeRawClient(ctx, config, client, endpoint) + return makeDockerClient(ctx, config, client, endpoint) } // Info show node info diff --git a/engine/docker/helper.go b/engine/docker/helper.go index 8df5fd70b..eda709c38 100644 --- a/engine/docker/helper.go +++ b/engine/docker/helper.go @@ -257,7 +257,7 @@ func GetIP(ctx context.Context, daemonHost string) string { return u.Hostname() } -func makeRawClient(_ context.Context, config coretypes.Config, client *http.Client, endpoint string) (engine.API, error) { +func makeDockerClient(_ context.Context, config coretypes.Config, client *http.Client, endpoint string) (engine.API, error) { cli, err := dockerapi.NewClientWithOpts( dockerapi.WithHost(endpoint), dockerapi.WithVersion(config.Docker.APIVersion), @@ -268,24 +268,6 @@ func makeRawClient(_ context.Context, config coretypes.Config, client *http.Clie return &Engine{cli, config}, nil } -func dumpFromString(ctx context.Context, ca, cert, key *os.File, caStr, certStr, keyStr string) error { - files := []*os.File{ca, cert, key} - data := []string{caStr, certStr, keyStr} - for i := 0; i < 3; i++ { - if _, err := files[i].WriteString(data[i]); err != nil { - return err - } - if err := files[i].Chmod(0444); err != nil { - return err - } - if err := files[i].Close(); err != nil { - return err - } - } - log.Debug(ctx, "[dumpFromString] Dump ca.pem, cert.pem, key.pem from string") - return nil -} - func useCNI(labels map[string]string) bool { for k, v := range labels { if k == "cni" && v == "1" { diff --git a/engine/factory/factory.go b/engine/factory/factory.go index 60dcf2ac6..d4e92fe50 100644 --- a/engine/factory/factory.go +++ b/engine/factory/factory.go @@ -139,11 +139,6 @@ func validateEngine(ctx context.Context, engine engine.API, timeout time.Duratio utils.WithTimeout(ctx, timeout, func(ctx context.Context) { err = engine.Ping(ctx) }) - if err != nil { - if closeErr := engine.CloseConn(); closeErr != nil { - log.Errorf(ctx, "[validateEngine] close conn error: %v", closeErr) - } - } return err } diff --git a/engine/virt/virt.go b/engine/virt/virt.go index fa0344b0e..482a7744c 100644 --- a/engine/virt/virt.go +++ b/engine/virt/virt.go @@ -90,7 +90,7 @@ func (v *Virt) CloseConn() error { func (v *Virt) Execute(ctx context.Context, ID string, config *enginetypes.ExecConfig) (pid string, stdout, stderr io.ReadCloser, stdin io.WriteCloser, err error) { if config.Tty { flags := virttypes.AttachGuestFlags{Safe: true, Force: true} - stream, err := v.client.AttachGuest(ctx, ID, config.Cmd, flags) + _, stream, err := v.client.AttachGuest(ctx, ID, config.Cmd, flags) if err != nil { return "", nil, nil, nil, err } @@ -274,7 +274,7 @@ func (v *Virt) VirtualizationLogs(ctx context.Context, opts *enginetypes.Virtual // VirtualizationAttach attaches something to a guest. func (v *Virt) VirtualizationAttach(ctx context.Context, ID string, stream, openStdin bool) (stdout, stderr io.ReadCloser, stdin io.WriteCloser, err error) { flags := virttypes.AttachGuestFlags{Safe: true, Force: true} - attachGuest, err := v.client.AttachGuest(ctx, ID, []string{}, flags) + _, attachGuest, err := v.client.AttachGuest(ctx, ID, []string{}, flags) if err != nil { return nil, nil, nil, err } diff --git a/go.mod b/go.mod index 1de442856..e3d969767 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/cenkalti/backoff/v4 v4.0.2 github.com/containerd/containerd v1.4.8 // indirect github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe // indirect + github.com/cornelk/hashmap v1.0.1 // indirect github.com/docker/distribution v2.7.1+incompatible github.com/docker/docker v20.10.0+incompatible github.com/docker/go-connections v0.4.0 @@ -32,7 +33,7 @@ require ( github.com/opencontainers/runc v1.0.0-rc95 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 - github.com/projecteru2/libyavirt v0.0.0-20220328133352-2a6302e90fc9 + github.com/projecteru2/libyavirt v0.0.0-20220329021434-9b7b732e9f8e github.com/prometheus/client_golang v1.11.0 github.com/sanity-io/litter v1.5.1 github.com/sirupsen/logrus v1.7.0 diff --git a/go.sum b/go.sum index 3aa266499..9942caaa0 100644 --- a/go.sum +++ b/go.sum @@ -107,6 +107,8 @@ github.com/coreos/go-systemd/v22 v22.3.1/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cornelk/hashmap v1.0.1 h1:RXGcy29hEdLLV8T6aK4s+BAd4tq4+3Hq50N2GoG0uIg= +github.com/cornelk/hashmap v1.0.1/go.mod h1:8wbysTUDnwJGrPZ1Iwsou3m+An6sldFrJItjRhfegCw= github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -119,6 +121,8 @@ github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dchest/siphash v1.1.0 h1:1Rs9eTUlZLPBEvV+2sTaM8O0NWn0ppbgqS7p11aWawI= +github.com/dchest/siphash v1.1.0/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= @@ -434,6 +438,8 @@ github.com/projecteru2/libyavirt v0.0.0-20211217082140-493b61aa9b0d h1:BMFqsvIB3 github.com/projecteru2/libyavirt v0.0.0-20211217082140-493b61aa9b0d/go.mod h1:FOc+hWBMLsMrmx5p3/moizKeSomedZPNwB6LhS+kEnE= github.com/projecteru2/libyavirt v0.0.0-20220328133352-2a6302e90fc9 h1:7atvx2788Nf+HvY5uZFLlogjplvvF7iw5nWI/8UkjoY= github.com/projecteru2/libyavirt v0.0.0-20220328133352-2a6302e90fc9/go.mod h1:FOc+hWBMLsMrmx5p3/moizKeSomedZPNwB6LhS+kEnE= +github.com/projecteru2/libyavirt v0.0.0-20220329021434-9b7b732e9f8e h1:IBRdnQybu4DRixX0grv4H4+tN8X+6Tp6LgVxx0ApIfM= +github.com/projecteru2/libyavirt v0.0.0-20220329021434-9b7b732e9f8e/go.mod h1:FOc+hWBMLsMrmx5p3/moizKeSomedZPNwB6LhS+kEnE= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= diff --git a/utils/http.go b/utils/http.go new file mode 100644 index 000000000..6ad3cc5ed --- /dev/null +++ b/utils/http.go @@ -0,0 +1,140 @@ +package utils + +import ( + "context" + "fmt" + "io/ioutil" + "net" + "net/http" + "os" + "runtime" + "strings" + "time" + + "github.com/cornelk/hashmap" + "github.com/docker/go-connections/tlsconfig" + + "github.com/projecteru2/core/log" +) + +var defaultHTTPClient = &http.Client{ + CheckRedirect: checkRedirect, + Transport: getDefaultTransport(), +} + +var defaultUnixSockClient = &http.Client{ + Transport: getDefaultUnixSockTransport(), +} + +var httpsClientCache = hashmap.New(32) + +// GetHTTPClient returns a HTTP client +func GetHTTPClient() *http.Client { + return defaultHTTPClient +} + +// GetUnixSockClient . +func GetUnixSockClient() *http.Client { + return defaultUnixSockClient +} + +// GetHTTPSClient returns an HTTPS client +// if cert_path/ca/cert/key is empty, it returns an HTTP client instead +func GetHTTPSClient(ctx context.Context, certPath, name, ca, cert, key string) (client *http.Client, err error) { + if certPath == "" || ca == "" || cert == "" || key == "" { + return GetHTTPClient(), nil + } + + cacheKey := name + SHA256(fmt.Sprintf("%s-%s-%s-%s-%s", certPath, name, ca, cert, key))[:8] + if httpsClient, ok := httpsClientCache.Get(cacheKey); ok { + return httpsClient.(*http.Client), nil + } + + caFile, err := ioutil.TempFile(certPath, fmt.Sprintf("ca-%s", name)) + if err != nil { + return nil, err + } + certFile, err := ioutil.TempFile(certPath, fmt.Sprintf("cert-%s", name)) + if err != nil { + return nil, err + } + keyFile, err := ioutil.TempFile(certPath, fmt.Sprintf("key-%s", name)) + if err != nil { + return nil, err + } + if err = dumpFromString(ctx, caFile, certFile, keyFile, ca, cert, key); err != nil { + return nil, err + } + options := tlsconfig.Options{ + CAFile: caFile.Name(), + CertFile: certFile.Name(), + KeyFile: keyFile.Name(), + InsecureSkipVerify: true, + } + defer os.Remove(caFile.Name()) + defer os.Remove(certFile.Name()) + defer os.Remove(keyFile.Name()) + tlsc, err := tlsconfig.Client(options) + if err != nil { + return nil, err + } + transport := getDefaultTransport() + transport.TLSClientConfig = tlsc + + client = &http.Client{ + CheckRedirect: checkRedirect, + Transport: transport, + } + httpsClientCache.Set(cacheKey, client) + return client, nil +} + +func getDefaultTransport() *http.Transport { + return &http.Transport{ + DialContext: (&net.Dialer{ + KeepAlive: time.Second * 30, + Timeout: time.Second * 30, + }).DialContext, + + IdleConnTimeout: time.Second * 90, + MaxIdleConnsPerHost: runtime.GOMAXPROCS(0) + 1, + Proxy: http.ProxyFromEnvironment, + } +} + +func getDefaultUnixSockTransport() *http.Transport { + return &http.Transport{ + DialContext: func(_ context.Context, _, addr string) (net.Conn, error) { + return net.DialTimeout("unix", strings.Split(addr, ":")[0], time.Second*30) + }, + + IdleConnTimeout: time.Second * 90, + MaxIdleConnsPerHost: runtime.GOMAXPROCS(0) + 1, + DisableCompression: true, + } +} + +func dumpFromString(ctx context.Context, ca, cert, key *os.File, caStr, certStr, keyStr string) error { + files := []*os.File{ca, cert, key} + data := []string{caStr, certStr, keyStr} + for i := 0; i < 3; i++ { + if _, err := files[i].WriteString(data[i]); err != nil { + return err + } + if err := files[i].Chmod(0444); err != nil { + return err + } + if err := files[i].Close(); err != nil { + return err + } + } + log.Debug(ctx, "[dumpFromString] Dump ca.pem, cert.pem, key.pem from string") + return nil +} + +func checkRedirect(req *http.Request, via []*http.Request) error { + if via[0].Method == http.MethodGet { + return http.ErrUseLastResponse + } + return fmt.Errorf("unexpected redirect") +}