Skip to content

Commit

Permalink
reuse http client (#570)
Browse files Browse the repository at this point in the history
  • Loading branch information
DuodenumL authored Mar 29, 2022
1 parent 02e6e8b commit 83b09a1
Show file tree
Hide file tree
Showing 7 changed files with 160 additions and 63 deletions.
45 changes: 9 additions & 36 deletions engine/docker/docker.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,16 @@ package docker

import (
"context"
"fmt"
"io/ioutil"
"net/http"
"os"
"strings"

"github.com/projecteru2/core/engine"
enginetypes "github.com/projecteru2/core/engine/types"
"github.com/projecteru2/core/log"
coretypes "github.com/projecteru2/core/types"
"github.com/projecteru2/core/utils"

dockerapi "github.com/docker/docker/client"
"github.com/docker/go-connections/tlsconfig"
)

const (
Expand All @@ -32,44 +30,19 @@ type Engine struct {
// MakeClient make docker cli
func MakeClient(ctx context.Context, config coretypes.Config, nodename, endpoint, ca, cert, key string) (engine.API, error) {
var client *http.Client
if config.CertPath != "" && ca != "" && cert != "" && key != "" { // nolint
caFile, err := ioutil.TempFile(config.CertPath, fmt.Sprintf("ca-%s", nodename))
var err error
if strings.HasPrefix(endpoint, "unix://") {
client = utils.GetUnixSockClient()
} else {
client, err = utils.GetHTTPSClient(ctx, config.CertPath, nodename, ca, cert, key)
if err != nil {
log.Errorf(ctx, "[MakeClient] GetHTTPSClient for %s %s error: %v", nodename, endpoint, err)
return nil, err
}
certFile, err := ioutil.TempFile(config.CertPath, fmt.Sprintf("cert-%s", nodename))
if err != nil {
return nil, err
}
keyFile, err := ioutil.TempFile(config.CertPath, fmt.Sprintf("key-%s", nodename))
if err != nil {
return nil, err
}
if err = dumpFromString(ctx, caFile, certFile, keyFile, ca, cert, key); err != nil {
return nil, err
}
options := tlsconfig.Options{
CAFile: caFile.Name(),
CertFile: certFile.Name(),
KeyFile: keyFile.Name(),
InsecureSkipVerify: true,
}
defer os.Remove(caFile.Name())
defer os.Remove(certFile.Name())
defer os.Remove(keyFile.Name())
tlsc, err := tlsconfig.Client(options)
if err != nil {
return nil, err
}
client = &http.Client{
Transport: &http.Transport{
TLSClientConfig: tlsc,
},
}
}

log.Debugf(ctx, "[MakeDockerEngine] Create new http.Client for %s, %s", endpoint, config.Docker.APIVersion)
return makeRawClient(ctx, config, client, endpoint)
return makeDockerClient(ctx, config, client, endpoint)
}

// Info show node info
Expand Down
20 changes: 1 addition & 19 deletions engine/docker/helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ func GetIP(ctx context.Context, daemonHost string) string {
return u.Hostname()
}

func makeRawClient(_ context.Context, config coretypes.Config, client *http.Client, endpoint string) (engine.API, error) {
func makeDockerClient(_ context.Context, config coretypes.Config, client *http.Client, endpoint string) (engine.API, error) {
cli, err := dockerapi.NewClientWithOpts(
dockerapi.WithHost(endpoint),
dockerapi.WithVersion(config.Docker.APIVersion),
Expand All @@ -268,24 +268,6 @@ func makeRawClient(_ context.Context, config coretypes.Config, client *http.Clie
return &Engine{cli, config}, nil
}

func dumpFromString(ctx context.Context, ca, cert, key *os.File, caStr, certStr, keyStr string) error {
files := []*os.File{ca, cert, key}
data := []string{caStr, certStr, keyStr}
for i := 0; i < 3; i++ {
if _, err := files[i].WriteString(data[i]); err != nil {
return err
}
if err := files[i].Chmod(0444); err != nil {
return err
}
if err := files[i].Close(); err != nil {
return err
}
}
log.Debug(ctx, "[dumpFromString] Dump ca.pem, cert.pem, key.pem from string")
return nil
}

func useCNI(labels map[string]string) bool {
for k, v := range labels {
if k == "cni" && v == "1" {
Expand Down
5 changes: 0 additions & 5 deletions engine/factory/factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -139,11 +139,6 @@ func validateEngine(ctx context.Context, engine engine.API, timeout time.Duratio
utils.WithTimeout(ctx, timeout, func(ctx context.Context) {
err = engine.Ping(ctx)
})
if err != nil {
if closeErr := engine.CloseConn(); closeErr != nil {
log.Errorf(ctx, "[validateEngine] close conn error: %v", closeErr)
}
}
return err
}

Expand Down
4 changes: 2 additions & 2 deletions engine/virt/virt.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ func (v *Virt) CloseConn() error {
func (v *Virt) Execute(ctx context.Context, ID string, config *enginetypes.ExecConfig) (pid string, stdout, stderr io.ReadCloser, stdin io.WriteCloser, err error) {
if config.Tty {
flags := virttypes.AttachGuestFlags{Safe: true, Force: true}
stream, err := v.client.AttachGuest(ctx, ID, config.Cmd, flags)
_, stream, err := v.client.AttachGuest(ctx, ID, config.Cmd, flags)
if err != nil {
return "", nil, nil, nil, err
}
Expand Down Expand Up @@ -274,7 +274,7 @@ func (v *Virt) VirtualizationLogs(ctx context.Context, opts *enginetypes.Virtual
// VirtualizationAttach attaches something to a guest.
func (v *Virt) VirtualizationAttach(ctx context.Context, ID string, stream, openStdin bool) (stdout, stderr io.ReadCloser, stdin io.WriteCloser, err error) {
flags := virttypes.AttachGuestFlags{Safe: true, Force: true}
attachGuest, err := v.client.AttachGuest(ctx, ID, []string{}, flags)
_, attachGuest, err := v.client.AttachGuest(ctx, ID, []string{}, flags)
if err != nil {
return nil, nil, nil, err
}
Expand Down
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ require (
github.com/cenkalti/backoff/v4 v4.0.2
github.com/containerd/containerd v1.4.8 // indirect
github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe // indirect
github.com/cornelk/hashmap v1.0.1 // indirect
github.com/docker/distribution v2.7.1+incompatible
github.com/docker/docker v20.10.0+incompatible
github.com/docker/go-connections v0.4.0
Expand All @@ -32,7 +33,7 @@ require (
github.com/opencontainers/runc v1.0.0-rc95 // indirect
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/pkg/errors v0.9.1
github.com/projecteru2/libyavirt v0.0.0-20220328133352-2a6302e90fc9
github.com/projecteru2/libyavirt v0.0.0-20220329021434-9b7b732e9f8e
github.com/prometheus/client_golang v1.11.0
github.com/sanity-io/litter v1.5.1
github.com/sirupsen/logrus v1.7.0
Expand Down
6 changes: 6 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,8 @@ github.com/coreos/go-systemd/v22 v22.3.1/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI=
github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cornelk/hashmap v1.0.1 h1:RXGcy29hEdLLV8T6aK4s+BAd4tq4+3Hq50N2GoG0uIg=
github.com/cornelk/hashmap v1.0.1/go.mod h1:8wbysTUDnwJGrPZ1Iwsou3m+An6sldFrJItjRhfegCw=
github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk=
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
Expand All @@ -119,6 +121,8 @@ github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dchest/siphash v1.1.0 h1:1Rs9eTUlZLPBEvV+2sTaM8O0NWn0ppbgqS7p11aWawI=
github.com/dchest/siphash v1.1.0/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4=
github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
Expand Down Expand Up @@ -434,6 +438,8 @@ github.com/projecteru2/libyavirt v0.0.0-20211217082140-493b61aa9b0d h1:BMFqsvIB3
github.com/projecteru2/libyavirt v0.0.0-20211217082140-493b61aa9b0d/go.mod h1:FOc+hWBMLsMrmx5p3/moizKeSomedZPNwB6LhS+kEnE=
github.com/projecteru2/libyavirt v0.0.0-20220328133352-2a6302e90fc9 h1:7atvx2788Nf+HvY5uZFLlogjplvvF7iw5nWI/8UkjoY=
github.com/projecteru2/libyavirt v0.0.0-20220328133352-2a6302e90fc9/go.mod h1:FOc+hWBMLsMrmx5p3/moizKeSomedZPNwB6LhS+kEnE=
github.com/projecteru2/libyavirt v0.0.0-20220329021434-9b7b732e9f8e h1:IBRdnQybu4DRixX0grv4H4+tN8X+6Tp6LgVxx0ApIfM=
github.com/projecteru2/libyavirt v0.0.0-20220329021434-9b7b732e9f8e/go.mod h1:FOc+hWBMLsMrmx5p3/moizKeSomedZPNwB6LhS+kEnE=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
Expand Down
140 changes: 140 additions & 0 deletions utils/http.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
package utils

import (
"context"
"fmt"
"io/ioutil"
"net"
"net/http"
"os"
"runtime"
"strings"
"time"

"github.com/cornelk/hashmap"
"github.com/docker/go-connections/tlsconfig"

"github.com/projecteru2/core/log"
)

var defaultHTTPClient = &http.Client{
CheckRedirect: checkRedirect,
Transport: getDefaultTransport(),
}

var defaultUnixSockClient = &http.Client{
Transport: getDefaultUnixSockTransport(),
}

var httpsClientCache = hashmap.New(32)

// GetHTTPClient returns a HTTP client
func GetHTTPClient() *http.Client {
return defaultHTTPClient
}

// GetUnixSockClient .
func GetUnixSockClient() *http.Client {
return defaultUnixSockClient
}

// GetHTTPSClient returns an HTTPS client
// if cert_path/ca/cert/key is empty, it returns an HTTP client instead
func GetHTTPSClient(ctx context.Context, certPath, name, ca, cert, key string) (client *http.Client, err error) {
if certPath == "" || ca == "" || cert == "" || key == "" {
return GetHTTPClient(), nil
}

cacheKey := name + SHA256(fmt.Sprintf("%s-%s-%s-%s-%s", certPath, name, ca, cert, key))[:8]
if httpsClient, ok := httpsClientCache.Get(cacheKey); ok {
return httpsClient.(*http.Client), nil
}

caFile, err := ioutil.TempFile(certPath, fmt.Sprintf("ca-%s", name))
if err != nil {
return nil, err
}
certFile, err := ioutil.TempFile(certPath, fmt.Sprintf("cert-%s", name))
if err != nil {
return nil, err
}
keyFile, err := ioutil.TempFile(certPath, fmt.Sprintf("key-%s", name))
if err != nil {
return nil, err
}
if err = dumpFromString(ctx, caFile, certFile, keyFile, ca, cert, key); err != nil {
return nil, err
}
options := tlsconfig.Options{
CAFile: caFile.Name(),
CertFile: certFile.Name(),
KeyFile: keyFile.Name(),
InsecureSkipVerify: true,
}
defer os.Remove(caFile.Name())
defer os.Remove(certFile.Name())
defer os.Remove(keyFile.Name())
tlsc, err := tlsconfig.Client(options)
if err != nil {
return nil, err
}
transport := getDefaultTransport()
transport.TLSClientConfig = tlsc

client = &http.Client{
CheckRedirect: checkRedirect,
Transport: transport,
}
httpsClientCache.Set(cacheKey, client)
return client, nil
}

func getDefaultTransport() *http.Transport {
return &http.Transport{
DialContext: (&net.Dialer{
KeepAlive: time.Second * 30,
Timeout: time.Second * 30,
}).DialContext,

IdleConnTimeout: time.Second * 90,
MaxIdleConnsPerHost: runtime.GOMAXPROCS(0) + 1,
Proxy: http.ProxyFromEnvironment,
}
}

func getDefaultUnixSockTransport() *http.Transport {
return &http.Transport{
DialContext: func(_ context.Context, _, addr string) (net.Conn, error) {
return net.DialTimeout("unix", strings.Split(addr, ":")[0], time.Second*30)
},

IdleConnTimeout: time.Second * 90,
MaxIdleConnsPerHost: runtime.GOMAXPROCS(0) + 1,
DisableCompression: true,
}
}

func dumpFromString(ctx context.Context, ca, cert, key *os.File, caStr, certStr, keyStr string) error {
files := []*os.File{ca, cert, key}
data := []string{caStr, certStr, keyStr}
for i := 0; i < 3; i++ {
if _, err := files[i].WriteString(data[i]); err != nil {
return err
}
if err := files[i].Chmod(0444); err != nil {
return err
}
if err := files[i].Close(); err != nil {
return err
}
}
log.Debug(ctx, "[dumpFromString] Dump ca.pem, cert.pem, key.pem from string")
return nil
}

func checkRedirect(req *http.Request, via []*http.Request) error {
if via[0].Method == http.MethodGet {
return http.ErrUseLastResponse
}
return fmt.Errorf("unexpected redirect")
}

0 comments on commit 83b09a1

Please sign in to comment.