From 190f35de8776b4cf70c6dc88608d139231734778 Mon Sep 17 00:00:00 2001 From: Saylor Berman Date: Wed, 4 May 2022 14:32:26 -0600 Subject: [PATCH] Update go-spiffe to v2 Updated the go-spiffe package to v2. Reworked the spiffe controller (now called spiffe cert fetcher) to use the new structure defined in this package to receive and write certs. --- go.mod | 9 ++- go.sum | 22 +++++-- internal/configs/configurator.go | 53 +++++++--------- internal/k8s/controller.go | 14 ++--- internal/k8s/spiffe.go | 105 +++++++++++++++++++------------ 5 files changed, 116 insertions(+), 87 deletions(-) diff --git a/go.mod b/go.mod index 6358c1c0d7..20a8d22302 100644 --- a/go.mod +++ b/go.mod @@ -13,8 +13,9 @@ require ( github.com/nginxinc/nginx-plus-go-client v0.9.0 github.com/nginxinc/nginx-prometheus-exporter v0.10.0 github.com/prometheus/client_golang v1.12.1 - github.com/spiffe/go-spiffe v1.1.0 + github.com/spiffe/go-spiffe/v2 v2.1.0 github.com/stretchr/testify v1.7.1 + google.golang.org/grpc v1.46.0 k8s.io/api v0.23.6 k8s.io/apimachinery v0.23.6 k8s.io/client-go v0.23.6 @@ -24,6 +25,7 @@ require ( ) require ( + github.com/Microsoft/go-winio v0.5.2 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/aws/aws-sdk-go-v2 v1.16.3 // indirect @@ -78,6 +80,7 @@ require ( github.com/rogpeppe/go-internal v1.6.1 // indirect github.com/spf13/cobra v1.3.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/zeebo/errs v1.2.2 // indirect go.etcd.io/etcd/api/v3 v3.5.1 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.1 // indirect go.etcd.io/etcd/client/v3 v3.5.0 // indirect @@ -107,9 +110,9 @@ require ( golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220118154757-00ab72f36ad5 // indirect - google.golang.org/grpc v1.43.0 // indirect - google.golang.org/protobuf v1.27.1 // indirect + google.golang.org/protobuf v1.28.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/square/go-jose.v2 v2.5.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect k8s.io/apiextensions-apiserver v0.23.4 // indirect diff --git a/go.sum b/go.sum index 4bc4b27e6c..035e00b439 100644 --- a/go.sum +++ b/go.sum @@ -64,6 +64,8 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -190,6 +192,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= @@ -628,8 +631,8 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= -github.com/spiffe/go-spiffe v1.1.0 h1:4GCqq8teavqkCl2j3c/vxQDgr33JidVRVT3mfssR6oM= -github.com/spiffe/go-spiffe v1.1.0/go.mod h1:HyNeJnVYkDyQgB2qcSPxVYkAA2F3lQu51bDxNpFcKxY= +github.com/spiffe/go-spiffe/v2 v2.1.0 h1:IZRlWhyFpPbJOiK8K+MwEFPU/QCdaW4Zf5bmIKBd3XM= +github.com/spiffe/go-spiffe/v2 v2.1.0/go.mod h1:5qg6rpqlwIub0JAiF1UK9IMD6BpPTmvG6yfSgDBs5lg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -657,6 +660,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/zeebo/errs v1.2.2 h1:5NFypMTuSdoySVTqlNs1dEoU21QVamMQJxW/Fii5O7g= +github.com/zeebo/errs v1.2.2/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= @@ -1119,6 +1124,7 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1161,7 +1167,6 @@ google.golang.org/genproto v0.0.0-20220118154757-00ab72f36ad5/go.mod h1:5CzLGKJ6 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -1186,9 +1191,10 @@ google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.43.0 h1:Eeu7bZtDZ2DpRCsLhUlcrLnvYaMK1Gz86a+hMVvELmM= -google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.46.0 h1:oCjezcn6g6A75TGoKYBPgKmVBLexhYLM6MebdrPApP8= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/grpc/examples v0.0.0-20201130180447-c456688b1860/go.mod h1:Ly7ZA/ARzg8fnPU9TyZIxoz33sEUuWX7txiqs8lPTgE= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1201,8 +1207,9 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1221,6 +1228,9 @@ gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXL gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.5.1 h1:7odma5RETjNHWJnR32wx8t+Io4djHE1PqxCFx3iiZ2w= +gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index 697cfbe05e..53e6c4332d 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -2,10 +2,7 @@ package configs import ( "bytes" - "crypto" - "crypto/x509" "encoding/json" - "encoding/pem" "fmt" "os" "strings" @@ -14,7 +11,7 @@ import ( "github.com/nginxinc/kubernetes-ingress/internal/k8s/secrets" "github.com/nginxinc/nginx-prometheus-exporter/collector" - "github.com/spiffe/go-spiffe/workload" + "github.com/spiffe/go-spiffe/v2/workloadapi" "github.com/nginxinc/kubernetes-ingress/internal/configs/version2" conf_v1alpha1 "github.com/nginxinc/kubernetes-ingress/pkg/apis/configuration/v1alpha1" @@ -125,7 +122,8 @@ type Configurator struct { // NewConfigurator creates a new Configurator. func NewConfigurator(nginxManager nginx.Manager, staticCfgParams *StaticConfigParams, config *ConfigParams, templateExecutor *version1.TemplateExecutor, templateExecutorV2 *version2.TemplateExecutor, isPlus bool, isWildcardEnabled bool, - labelUpdater collector.LabelUpdater, isPrometheusEnabled bool, latencyCollector latCollector.LatencyCollector, isLatencyMetricsEnabled bool) *Configurator { + labelUpdater collector.LabelUpdater, isPrometheusEnabled bool, latencyCollector latCollector.LatencyCollector, isLatencyMetricsEnabled bool, +) *Configurator { metricLabelsIndex := &metricLabelsIndex{ ingressUpstreams: make(map[string][]string), virtualServerUpstreams: make(map[string][]string), @@ -1241,41 +1239,33 @@ func (cnf *Configurator) GetVirtualServerCounts() (vsCount int, vsrCount int) { } // AddOrUpdateSpiffeCerts writes Spiffe certs and keys to disk and reloads NGINX -func (cnf *Configurator) AddOrUpdateSpiffeCerts(svidResponse *workload.X509SVIDs) error { - svid := svidResponse.Default() - privateKeyBytes, err := x509.MarshalPKCS8PrivateKey(svid.PrivateKey.(crypto.PrivateKey)) +func (cnf *Configurator) AddOrUpdateSpiffeCerts(svidResponse *workloadapi.X509Context) error { + svid := svidResponse.DefaultSVID() + trustDomain := svid.ID.TrustDomain() + caBundle, err := svidResponse.Bundles.GetX509BundleForTrustDomain(trustDomain) if err != nil { - return fmt.Errorf("error when marshaling private key: %w", err) + return fmt.Errorf("error parsing CA bundle from SPIFFE SVID response: %w", err) } - cnf.nginxManager.CreateSecret(spiffeKeyFileName, createSpiffeKey(privateKeyBytes), spiffeKeyFileMode) - cnf.nginxManager.CreateSecret(spiffeCertFileName, createSpiffeCert(svid.Certificates), spiffeCertsFileMode) - cnf.nginxManager.CreateSecret(spiffeBundleFileName, createSpiffeCert(svid.TrustBundle), spiffeCertsFileMode) + pemBundle, err := caBundle.Marshal() + if err != nil { + return fmt.Errorf("unable to marshal X.509 SVID Bundle: %w", err) + } - err = cnf.reload(nginx.ReloadForOtherUpdate) + pemCerts, pemKey, err := svid.Marshal() if err != nil { - return fmt.Errorf("error when reloading NGINX when updating the SPIFFE Certs: %w", err) + return fmt.Errorf("unable to marshal X.509 SVID: %w", err) } - return nil -} -func createSpiffeKey(content []byte) []byte { - return pem.EncodeToMemory(&pem.Block{ - Type: "EC PRIVATE KEY", - Bytes: content, - }) -} + cnf.nginxManager.CreateSecret(spiffeKeyFileName, pemKey, spiffeKeyFileMode) + cnf.nginxManager.CreateSecret(spiffeCertFileName, pemCerts, spiffeCertsFileMode) + cnf.nginxManager.CreateSecret(spiffeBundleFileName, pemBundle, spiffeCertsFileMode) -func createSpiffeCert(certs []*x509.Certificate) []byte { - pemData := make([]byte, 0, len(certs)) - for _, c := range certs { - b := &pem.Block{ - Type: "CERTIFICATE", - Bytes: c.Raw, - } - pemData = append(pemData, pem.EncodeToMemory(b)...) + err = cnf.reload(nginx.ReloadForOtherUpdate) + if err != nil { + return fmt.Errorf("error when reloading NGINX when updating the SPIFFE Certs: %w", err) } - return pemData + return nil } func (cnf *Configurator) updateApResources(ingEx *IngressEx) *AppProtectResources { @@ -1444,7 +1434,6 @@ func (cnf *Configurator) DeleteAppProtectLogConf(resource *unstructured.Unstruct func (cnf *Configurator) RefreshAppProtectUserSigs( userSigs []*unstructured.Unstructured, delPols []string, ingExes []*IngressEx, mergeableIngresses []*MergeableIngresses, vsExes []*VirtualServerEx, ) (Warnings, error) { - allWarnings, err := cnf.addOrUpdateIngressesAndVirtualServers(ingExes, mergeableIngresses, vsExes) if err != nil { return allWarnings, err diff --git a/internal/k8s/controller.go b/internal/k8s/controller.go index ed8762e3a5..2c79db23d5 100644 --- a/internal/k8s/controller.go +++ b/internal/k8s/controller.go @@ -35,7 +35,7 @@ import ( "github.com/golang/glog" "github.com/nginxinc/kubernetes-ingress/internal/k8s/secrets" - "github.com/spiffe/go-spiffe/workload" + "github.com/spiffe/go-spiffe/v2/workloadapi" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" @@ -152,7 +152,7 @@ type LoadBalancerController struct { metricsCollector collectors.ControllerCollector globalConfigurationValidator *validation.GlobalConfigurationValidator transportServerValidator *validation.TransportServerValidator - spiffeController *SpiffeController + spiffeCertFetcher *SpiffeCertFetcher internalRoutesEnabled bool syncLock sync.Mutex isNginxReady bool @@ -247,7 +247,7 @@ func NewLoadBalancerController(input NewLoadBalancerControllerInput) *LoadBalanc lbc.syncQueue = newTaskQueue(lbc.sync) if input.SpireAgentAddress != "" { var err error - lbc.spiffeController, err = NewSpiffeController(lbc.syncSVIDRotation, input.SpireAgentAddress) + lbc.spiffeCertFetcher, err = NewSpiffeCertFetcher(lbc.syncSVIDRotation, input.SpireAgentAddress) if err != nil { glog.Fatalf("failed to create Spiffe Controller: %v", err) } @@ -544,8 +544,8 @@ func (lbc *LoadBalancerController) addIngressLinkHandler(handlers cache.Resource func (lbc *LoadBalancerController) Run() { lbc.ctx, lbc.cancel = context.WithCancel(context.Background()) - if lbc.spiffeController != nil { - err := lbc.spiffeController.Start(lbc.ctx.Done(), lbc.addInternalRouteServer) + if lbc.spiffeCertFetcher != nil { + err := lbc.spiffeCertFetcher.Start(lbc.ctx, lbc.addInternalRouteServer) if err != nil { glog.Fatal(err) } @@ -768,7 +768,7 @@ func (lbc *LoadBalancerController) preSyncSecrets() { func (lbc *LoadBalancerController) sync(task task) { glog.V(3).Infof("Syncing %v", task.Key) - if lbc.spiffeController != nil { + if lbc.spiffeCertFetcher != nil { lbc.syncLock.Lock() defer lbc.syncLock.Unlock() } @@ -3279,7 +3279,7 @@ func formatWarningMessages(w []string) string { return strings.Join(w, "; ") } -func (lbc *LoadBalancerController) syncSVIDRotation(svidResponse *workload.X509SVIDs) { +func (lbc *LoadBalancerController) syncSVIDRotation(svidResponse *workloadapi.X509Context) { lbc.syncLock.Lock() defer lbc.syncLock.Unlock() glog.V(3).Info("Rotating SPIFFE Certificates") diff --git a/internal/k8s/spiffe.go b/internal/k8s/spiffe.go index dfb7cfcfb6..cdcb439d7e 100644 --- a/internal/k8s/spiffe.go +++ b/internal/k8s/spiffe.go @@ -1,45 +1,67 @@ package k8s import ( + "context" "errors" "fmt" - "strings" "time" "github.com/golang/glog" - "github.com/spiffe/go-spiffe/workload" + "github.com/spiffe/go-spiffe/v2/workloadapi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) -// SpiffeController controls spiffe -type SpiffeController struct { - watcher *spiffeWatcher - client *workload.X509SVIDClient +// Client wraps the workloadapi.Client +type Client interface { + WatchX509Context(context.Context, workloadapi.X509ContextWatcher) error + Close() error } -// NewSpiffeController creates the spiffeWatcher and the Spiffe Workload API Client, +// SpiffeCertFetcher fetches certs from the X509 SPIFFE Workload API. +type SpiffeCertFetcher struct { + client Client + watcher *spiffeWatcher + watchErrCh chan error +} + +// NewSpiffeCertFetcher creates the spiffeWatcher and the Spiffe Workload API Client, // returns an error if the client cannot connect to the Spire Agent. -func NewSpiffeController(sync func(*workload.X509SVIDs), spireAgentAddr string) (*SpiffeController, error) { - watcher := &spiffeWatcher{sync: sync} - client, err := workload.NewX509SVIDClient(watcher, workload.WithAddr("unix://"+spireAgentAddr)) +func NewSpiffeCertFetcher(sync func(*workloadapi.X509Context), spireAgentAddr string) (*SpiffeCertFetcher, error) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + client, err := workloadapi.New(ctx, workloadapi.WithAddr("unix://"+spireAgentAddr)) if err != nil { - return nil, fmt.Errorf("failed to create Spiffe Workload API Client: %w", err) + return nil, fmt.Errorf("could not create SPIFFE Workload API Client: %w", err) } - sc := &SpiffeController{ - watcher: watcher, - client: client, - } - return sc, nil + + return &SpiffeCertFetcher{ + watchErrCh: make(chan error), + client: client, + watcher: &spiffeWatcher{sync: sync}, + }, nil } // Start starts the Spiffe Workload API Client and waits for the Spiffe certs to be written to disk. // If the certs are not available after 30 seconds an error is returned. -// On success, calls onStart function and kicks off the Spiffe Controller's run loop. -func (sc *SpiffeController) Start(stopCh <-chan struct{}, onStart func()) error { +// On success, calls onStart function and kicks off the SpiffeCertFetcher's run loop. +func (sc *SpiffeCertFetcher) Start(ctx context.Context, onStart func()) error { glog.V(3).Info("Starting SPIFFE Workload API Client") - err := sc.client.Start() - if err != nil { - return fmt.Errorf("failed to start Spiffe Workload API Client: %w", err) - } + + go func() { + defer func() { + if err := sc.client.Close(); err != nil && status.Code(err) != codes.Canceled { + glog.V(3).Info("error closing SPIFFE Workload API Client: ", err) + } + }() + err := sc.client.WatchX509Context(ctx, sc.watcher) + if err != nil && status.Code(err) != codes.Canceled { + sc.watchErrCh <- err + } + }() + + stopCh := ctx.Done() timeout := time.After(30 * time.Second) duration := 100 * time.Millisecond for { @@ -50,8 +72,10 @@ func (sc *SpiffeController) Start(stopCh <-chan struct{}, onStart func()) error select { case <-timeout: return errors.New("timed out waiting for SPIFFE trust bundle") + case err := <-sc.watchErrCh: + return fmt.Errorf("error waiting for initial trust bundle: %w", err) case <-stopCh: - return sc.client.Stop() + return sc.client.Close() default: break } @@ -63,34 +87,37 @@ func (sc *SpiffeController) Start(stopCh <-chan struct{}, onStart func()) error } // Run waits until a message is sent on the stop channel and stops the Spiffe Workload API Client. -func (sc *SpiffeController) Run(stopCh <-chan struct{}) { +func (sc *SpiffeCertFetcher) Run(stopCh <-chan struct{}) { <-stopCh - err := sc.client.Stop() - if err != nil { + if err := sc.client.Close(); err != nil { glog.Errorf("failed to stop Spiffe Workload API Client: %v", err) } } -// spiffeWatcher is a sample implementation of the workload.X509SVIDWatcher interface +// spiffeWatcher is a sample implementation of the workload.X509ContextWatcher interface type spiffeWatcher struct { - sync func(*workload.X509SVIDs) + sync func(*workloadapi.X509Context) synced bool } -// UpdateX509SVIDs is run every time an SVID is updated -func (w *spiffeWatcher) UpdateX509SVIDs(svids *workload.X509SVIDs) { - for _, svid := range svids.SVIDs { - glog.V(3).Infof("SVID updated for spiffeID: %q", svid.SPIFFEID) - } - w.sync(svids) +// OnX509ContextUpdate is called when a new X.509 Context is fetched from the SPIFFE Workload API. +func (w *spiffeWatcher) OnX509ContextUpdate(svidResponse *workloadapi.X509Context) { + glog.V(3).Infof("SVID updated for for spiffeID: %q\n", svidResponse.DefaultSVID().ID) + w.sync(svidResponse) w.synced = true } -// OnError is run when the client runs into an error -func (w *spiffeWatcher) OnError(err error) { - if strings.Contains(err.Error(), "PermissionDenied") { - glog.V(3).Infof("X509SVIDClient still waiting for certificates: %v", err) +// OnX509WatchError is called when there is an error watching the X.509 Contexts from the SPIFFE Workload API. +func (w *spiffeWatcher) OnX509ContextWatchError(err error) { + msg := "For more information check the logs of the Spire agents and server." + switch status.Code(err) { + case codes.Unavailable: + glog.V(3).Infof("X509SVIDClient cannot connect to the Spire agent: %v. %s", err, msg) + case codes.PermissionDenied: + glog.V(3).Infof("X509SVIDClient still waiting for certificates: %v. %s", err, msg) + case codes.Canceled: return + default: + glog.V(3).Infof("X509SVIDClient error: %v. %s", err, msg) } - glog.Fatal(err) }