From 3ec1b94020e02ae3d841095a83a488e9523d4df6 Mon Sep 17 00:00:00 2001 From: Garrybest Date: Mon, 7 Nov 2022 19:58:40 +0800 Subject: [PATCH 01/87] get kubelet config from configz Signed-off-by: Garrybest --- cmd/nfd-topology-updater/main.go | 50 +++++++++-- cmd/nfd-topology-updater/main_test.go | 17 ++-- .../topologyupdater-clusterrole.yaml | 6 ++ .../topologyupdater-mounts.yaml | 4 - .../templates/clusterrole.yaml | 6 ++ .../topology-updater-commandline-reference.md | 26 ++++-- pkg/resourcemonitor/types.go | 3 +- .../kubeconf/kubelet_config_file.go | 0 .../kubeconf/kubelet_config_file_test.go | 2 +- pkg/utils/kubeconf/kubelet_configz.go | 83 +++++++++++++++++++ 10 files changed, 171 insertions(+), 26 deletions(-) rename pkg/{ => utils}/kubeconf/kubelet_config_file.go (100%) rename pkg/{ => utils}/kubeconf/kubelet_config_file_test.go (93%) create mode 100644 pkg/utils/kubeconf/kubelet_configz.go diff --git a/cmd/nfd-topology-updater/main.go b/cmd/nfd-topology-updater/main.go index 0f8872ef75..bdb9b7c924 100644 --- a/cmd/nfd-topology-updater/main.go +++ b/cmd/nfd-topology-updater/main.go @@ -19,23 +19,27 @@ package main import ( "flag" "fmt" + "net/url" "os" "time" "k8s.io/klog/v2" + kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" - "sigs.k8s.io/node-feature-discovery/pkg/kubeconf" + nfdclient "sigs.k8s.io/node-feature-discovery/pkg/nfd-client" topology "sigs.k8s.io/node-feature-discovery/pkg/nfd-client/topology-updater" "sigs.k8s.io/node-feature-discovery/pkg/resourcemonitor" "sigs.k8s.io/node-feature-discovery/pkg/topologypolicy" "sigs.k8s.io/node-feature-discovery/pkg/utils" "sigs.k8s.io/node-feature-discovery/pkg/utils/hostpath" + "sigs.k8s.io/node-feature-discovery/pkg/utils/kubeconf" "sigs.k8s.io/node-feature-discovery/pkg/version" ) const ( // ProgramName is the canonical name of this program - ProgramName = "nfd-topology-updater" + ProgramName = "nfd-topology-updater" + kubeletSecurePort = 10250 ) func main() { @@ -58,10 +62,33 @@ func main() { // Plug klog into grpc logging infrastructure utils.ConfigureGrpcKlog() - klConfig, err := kubeconf.GetKubeletConfigFromLocalFile(resourcemonitorArgs.KubeletConfigFile) + u, err := url.ParseRequestURI(resourcemonitorArgs.KubeletConfigURI) if err != nil { - klog.Exitf("error reading kubelet config: %v", err) + klog.Exitf("failed to parse args for kubelet-config-uri: %v", err) } + + // init kubelet API client + var klConfig *kubeletconfigv1beta1.KubeletConfiguration + switch u.Scheme { + case "file": + klConfig, err = kubeconf.GetKubeletConfigFromLocalFile(u.Path) + if err != nil { + klog.Exitf("failed to read kubelet config: %v", err) + } + case "https": + restConfig, err := kubeconf.InsecureConfig(u.String(), resourcemonitorArgs.APIAuthTokenFile) + if err != nil { + klog.Exitf("failed to initialize rest config for kubelet config uri: %v", err) + } + + klConfig, err = kubeconf.GetKubeletConfiguration(restConfig) + if err != nil { + klog.Exitf("failed to get kubelet config from configz endpoint: %v", err) + } + default: + klog.Exitf("unsupported URI scheme: %v", u.Scheme) + } + tmPolicy := string(topologypolicy.DetectTopologyPolicy(klConfig.TopologyManagerPolicy, klConfig.TopologyManagerScope)) klog.Infof("detected kubelet Topology Manager policy %q", tmPolicy) @@ -86,6 +113,15 @@ func parseArgs(flags *flag.FlagSet, osArgs ...string) (*topology.Args, *resource os.Exit(2) } + if len(resourcemonitorArgs.KubeletConfigURI) == 0 { + if len(nfdclient.NodeName()) == 0 { + fmt.Fprintf(flags.Output(), "unable to determine the default kubelet config endpoint 'https://${NODE_NAME}:%d/configz' due to empty NODE_NAME environment, "+ + "please either define the NODE_NAME environment variable or specify endpoint with the -kubelet-config-uri flag\n", kubeletSecurePort) + os.Exit(1) + } + resourcemonitorArgs.KubeletConfigURI = fmt.Sprintf("https://%s:%d/configz", nfdclient.NodeName(), kubeletSecurePort) + } + return args, resourcemonitorArgs } @@ -109,8 +145,10 @@ func initFlags(flagset *flag.FlagSet) (*topology.Args, *resourcemonitor.Args) { "Time to sleep between CR updates. Non-positive value implies no CR updatation (i.e. infinite sleep). [Default: 60s]") flagset.StringVar(&resourcemonitorArgs.Namespace, "watch-namespace", "*", "Namespace to watch pods (for testing/debugging purpose). Use * for all namespaces.") - flagset.StringVar(&resourcemonitorArgs.KubeletConfigFile, "kubelet-config-file", hostpath.VarDir.Path("lib/kubelet/config.yaml"), - "Kubelet config file path.") + flagset.StringVar(&resourcemonitorArgs.KubeletConfigURI, "kubelet-config-uri", "", + "Kubelet config URI path. Default to kubelet configz endpoint.") + flagset.StringVar(&resourcemonitorArgs.APIAuthTokenFile, "api-auth-token-file", "/var/run/secrets/kubernetes.io/serviceaccount/token", + "API auth token file path. It is used to request kubelet configz endpoint, only takes effect when kubelet-config-uri is https. Default to /var/run/secrets/kubernetes.io/serviceaccount/token.") flagset.StringVar(&resourcemonitorArgs.PodResourceSocketPath, "podresources-socket", hostpath.VarDir.Path("lib/kubelet/pod-resources/kubelet.sock"), "Pod Resource Socket path to use.") flagset.StringVar(&args.Server, "server", "localhost:8080", diff --git a/cmd/nfd-topology-updater/main_test.go b/cmd/nfd-topology-updater/main_test.go index 9094bb0231..e1b249808a 100644 --- a/cmd/nfd-topology-updater/main_test.go +++ b/cmd/nfd-topology-updater/main_test.go @@ -29,33 +29,33 @@ func TestArgsParse(t *testing.T) { flags := flag.NewFlagSet(ProgramName, flag.ExitOnError) Convey("When -no-publish and -oneshot flags are passed", func() { - args, finderArgs := parseArgs(flags, "-oneshot", "-no-publish") + args, finderArgs := parseArgs(flags, "-oneshot", "-no-publish", "-kubelet-config-uri=https://%s:%d/configz") Convey("noPublish is set and args.sources is set to the default value", func() { So(args.NoPublish, ShouldBeTrue) So(args.Oneshot, ShouldBeTrue) So(finderArgs.SleepInterval, ShouldEqual, 60*time.Second) - So(finderArgs.KubeletConfigFile, ShouldEqual, "/var/lib/kubelet/config.yaml") So(finderArgs.PodResourceSocketPath, ShouldEqual, "/var/lib/kubelet/pod-resources/kubelet.sock") }) }) - Convey("When valid args are specified for -kubelet-config-file and -sleep-interval,", func() { + Convey("When valid args are specified for -kubelet-config-url and -sleep-interval,", func() { args, finderArgs := parseArgs(flags, - "-kubelet-config-file=/path/testconfig.yaml", + "-kubelet-config-uri=file:///path/testconfig.yaml", "-sleep-interval=30s") Convey("args.sources is set to appropriate values", func() { So(args.NoPublish, ShouldBeFalse) So(args.Oneshot, ShouldBeFalse) So(finderArgs.SleepInterval, ShouldEqual, 30*time.Second) - So(finderArgs.KubeletConfigFile, ShouldEqual, "/path/testconfig.yaml") + So(finderArgs.KubeletConfigURI, ShouldEqual, "file:///path/testconfig.yaml") So(finderArgs.PodResourceSocketPath, ShouldEqual, "/var/lib/kubelet/pod-resources/kubelet.sock") }) }) Convey("When valid args are specified for -podresources-socket flag and -sleep-interval is specified", func() { args, finderArgs := parseArgs(flags, + "-kubelet-config-uri=https://%s:%d/configz", "-podresources-socket=/path/testkubelet.sock", "-sleep-interval=30s") @@ -63,19 +63,18 @@ func TestArgsParse(t *testing.T) { So(args.NoPublish, ShouldBeFalse) So(args.Oneshot, ShouldBeFalse) So(finderArgs.SleepInterval, ShouldEqual, 30*time.Second) - So(finderArgs.KubeletConfigFile, ShouldEqual, "/var/lib/kubelet/config.yaml") So(finderArgs.PodResourceSocketPath, ShouldEqual, "/path/testkubelet.sock") }) }) Convey("When valid -sleep-inteval is specified", func() { args, finderArgs := parseArgs(flags, + "-kubelet-config-uri=https://%s:%d/configz", "-sleep-interval=30s") Convey("args.sources is set to appropriate values", func() { So(args.NoPublish, ShouldBeFalse) So(args.Oneshot, ShouldBeFalse) So(finderArgs.SleepInterval, ShouldEqual, 30*time.Second) - So(finderArgs.KubeletConfigFile, ShouldEqual, "/var/lib/kubelet/config.yaml") So(finderArgs.PodResourceSocketPath, ShouldEqual, "/var/lib/kubelet/pod-resources/kubelet.sock") }) }) @@ -84,7 +83,7 @@ func TestArgsParse(t *testing.T) { args, finderArgs := parseArgs(flags, "-no-publish", "-sleep-interval=30s", - "-kubelet-config-file=/path/testconfig.yaml", + "-kubelet-config-uri=file:///path/testconfig.yaml", "-podresources-socket=/path/testkubelet.sock", "-ca-file=ca", "-cert-file=crt", @@ -96,7 +95,7 @@ func TestArgsParse(t *testing.T) { So(args.CertFile, ShouldEqual, "crt") So(args.KeyFile, ShouldEqual, "key") So(finderArgs.SleepInterval, ShouldEqual, 30*time.Second) - So(finderArgs.KubeletConfigFile, ShouldEqual, "/path/testconfig.yaml") + So(finderArgs.KubeletConfigURI, ShouldEqual, "file:///path/testconfig.yaml") So(finderArgs.PodResourceSocketPath, ShouldEqual, "/path/testkubelet.sock") }) }) diff --git a/deployment/base/rbac-topologyupdater/topologyupdater-clusterrole.yaml b/deployment/base/rbac-topologyupdater/topologyupdater-clusterrole.yaml index 35de71b10d..1719edbe06 100644 --- a/deployment/base/rbac-topologyupdater/topologyupdater-clusterrole.yaml +++ b/deployment/base/rbac-topologyupdater/topologyupdater-clusterrole.yaml @@ -10,6 +10,12 @@ rules: verbs: - get - list +- apiGroups: + - "" + resources: + - nodes/proxy + verbs: + - get - apiGroups: - "" resources: diff --git a/deployment/components/topology-updater/topologyupdater-mounts.yaml b/deployment/components/topology-updater/topologyupdater-mounts.yaml index c792c99e92..89f1c2e5e2 100644 --- a/deployment/components/topology-updater/topologyupdater-mounts.yaml +++ b/deployment/components/topology-updater/topologyupdater-mounts.yaml @@ -21,10 +21,6 @@ - name: host-sys mountPath: /host-sys -- op: add - path: /spec/template/spec/containers/0/args/- - value: "-kubelet-config-file=/host-var/lib/kubelet/config.yaml" - - op: add path: /spec/template/spec/containers/0/args/- value: "-podresources-socket=/host-var/lib/kubelet/pod-resources/kubelet.sock" diff --git a/deployment/helm/node-feature-discovery/templates/clusterrole.yaml b/deployment/helm/node-feature-discovery/templates/clusterrole.yaml index 36a12ecbe7..587a5a9554 100644 --- a/deployment/helm/node-feature-discovery/templates/clusterrole.yaml +++ b/deployment/helm/node-feature-discovery/templates/clusterrole.yaml @@ -18,6 +18,12 @@ rules: - patch - update - list +- apiGroups: + - "" + resources: + - nodes/proxy + verbs: + - get - apiGroups: - nfd.k8s-sigs.io resources: diff --git a/docs/reference/topology-updater-commandline-reference.md b/docs/reference/topology-updater-commandline-reference.md index e872e8d852..742d36c66d 100644 --- a/docs/reference/topology-updater-commandline-reference.md +++ b/docs/reference/topology-updater-commandline-reference.md @@ -169,17 +169,33 @@ Example: nfd-topology-updater -watch-namespace=rte ``` -### -kubelet-config-file +### -kubelet-config-uri -The `-kubelet-config-file` specifies the path to the Kubelet's configuration -file. +The `-kubelet-config-uri` specifies the path to the Kubelet's configuration. +Note that the URi could either be a local host file or an HTTP endpoint. -Default: /host-var/lib/kubelet/config.yaml +Default: `https://${NODE_NAME}:10250/configz` Example: ```bash -nfd-topology-updater -kubelet-config-file=/var/lib/kubelet/config.yaml +nfd-topology-updater -kubelet-config-uri=file:///var/lib/kubelet/config.yaml +``` + +### -api-auth-token-file + +The `-api-auth-token-file` specifies the path to the api auth token file +which is used to retrieve Kubelet's configuration from Kubelet secure port, +only taking effect when `-kubelet-config-uri` is https. +Note that this token file must bind to a role that has the `get` capability to +`nodes/proxy` resources. + +Default: `/var/run/secrets/kubernetes.io/serviceaccount/token` + +Example: + +```bash +nfd-topology-updater -token-file=/var/run/secrets/kubernetes.io/serviceaccount/token ``` ### -podresources-socket diff --git a/pkg/resourcemonitor/types.go b/pkg/resourcemonitor/types.go index b288ef8c71..5b0a4c5feb 100644 --- a/pkg/resourcemonitor/types.go +++ b/pkg/resourcemonitor/types.go @@ -29,7 +29,8 @@ type Args struct { PodResourceSocketPath string SleepInterval time.Duration Namespace string - KubeletConfigFile string + KubeletConfigURI string + APIAuthTokenFile string } // ResourceInfo stores information of resources and their corresponding IDs obtained from PodResource API diff --git a/pkg/kubeconf/kubelet_config_file.go b/pkg/utils/kubeconf/kubelet_config_file.go similarity index 100% rename from pkg/kubeconf/kubelet_config_file.go rename to pkg/utils/kubeconf/kubelet_config_file.go diff --git a/pkg/kubeconf/kubelet_config_file_test.go b/pkg/utils/kubeconf/kubelet_config_file_test.go similarity index 93% rename from pkg/kubeconf/kubelet_config_file_test.go rename to pkg/utils/kubeconf/kubelet_config_file_test.go index e2891a4a04..6220561171 100644 --- a/pkg/kubeconf/kubelet_config_file_test.go +++ b/pkg/utils/kubeconf/kubelet_config_file_test.go @@ -29,7 +29,7 @@ type testCaseData struct { func TestGetKubeletConfigFromLocalFile(t *testing.T) { tCases := []testCaseData{ { - path: filepath.Join("..", "..", "test", "data", "kubeletconf.yaml"), + path: filepath.Join("..", "..", "..", "test", "data", "kubeletconf.yaml"), tmPolicy: "single-numa-node", }, } diff --git a/pkg/utils/kubeconf/kubelet_configz.go b/pkg/utils/kubeconf/kubelet_configz.go new file mode 100644 index 0000000000..a7791f5de4 --- /dev/null +++ b/pkg/utils/kubeconf/kubelet_configz.go @@ -0,0 +1,83 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeconf + +import ( + "context" + "encoding/json" + "fmt" + "os" + "time" + + "k8s.io/client-go/discovery" + "k8s.io/client-go/rest" + kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" +) + +// GetKubeletConfiguration returns the kubelet configuration. +func GetKubeletConfiguration(restConfig *rest.Config) (*kubeletconfigv1beta1.KubeletConfiguration, error) { + discoveryClient, err := discovery.NewDiscoveryClientForConfig(restConfig) + if err != nil { + return nil, err + } + + var timeout time.Duration + // This hack because /configz reports the following structure: + // {"kubeletconfig": {the JSON representation of kubeletconfigv1beta1.KubeletConfiguration}} + type configzWrapper struct { + ComponentConfig kubeletconfigv1beta1.KubeletConfiguration `json:"kubeletconfig"` + } + bytes, err := discoveryClient.RESTClient(). + Get(). + Timeout(timeout). + Do(context.TODO()). + Raw() + if err != nil { + return nil, err + } + + configz := configzWrapper{} + if err = json.Unmarshal(bytes, &configz); err != nil { + return nil, fmt.Errorf("failed to unmarshal json for kubelet config: %w", err) + } + + return &configz.ComponentConfig, nil +} + +// InsecureConfig returns a kubelet API config object which uses the token path. +func InsecureConfig(host, tokenFile string) (*rest.Config, error) { + if tokenFile == "" { + return nil, fmt.Errorf("api auth token file must be defined") + } + if len(host) == 0 { + return nil, fmt.Errorf("kubelet host must be defined") + } + + token, err := os.ReadFile(tokenFile) + if err != nil { + return nil, err + } + + tlsClientConfig := rest.TLSClientConfig{Insecure: true} + + return &rest.Config{ + Host: host, + TLSClientConfig: tlsClientConfig, + BearerToken: string(token), + BearerTokenFile: tokenFile, + }, nil +} From 6f49421c0e5ccb7ebde0ba19d7c5f4817f24c4e1 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Wed, 16 Nov 2022 21:08:13 +0200 Subject: [PATCH 02/87] docs: update github-pages gem to v227 --- docs/Gemfile | 2 +- docs/Gemfile.lock | 208 +++++++++++++++++++++------------------------- docs/_config.yml | 2 + 3 files changed, 99 insertions(+), 113 deletions(-) diff --git a/docs/Gemfile b/docs/Gemfile index 129a2e6499..a9df3aa1b5 100644 --- a/docs/Gemfile +++ b/docs/Gemfile @@ -12,7 +12,7 @@ gem "jekyll", "~> 3.9.0" # If you want to use GitHub Pages, remove the "gem "jekyll"" above and # uncomment the line below. To upgrade, run `bundle update github-pages`. -gem "github-pages", "~> 209", group: :jekyll_plugins +gem "github-pages", "~> 227", group: :jekyll_plugins # Windows does not include zoneinfo files, so bundle the tzinfo-data gem # and associated library. diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock index 511bce571b..bc9f115244 100644 --- a/docs/Gemfile.lock +++ b/docs/Gemfile.lock @@ -1,110 +1,95 @@ GEM remote: https://rubygems.org/ specs: - activesupport (6.0.4.1) + activesupport (6.0.6) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) tzinfo (~> 1.1) zeitwerk (~> 2.2, >= 2.2.2) - addressable (2.8.0) - public_suffix (>= 2.0.2, < 5.0) + addressable (2.8.1) + public_suffix (>= 2.0.2, < 6.0) coffee-script (2.4.1) coffee-script-source execjs coffee-script-source (1.11.1) colorator (1.1.0) - commonmarker (0.17.13) - ruby-enum (~> 0.5) - concurrent-ruby (1.1.9) - dnsruby (1.61.7) + commonmarker (0.23.6) + concurrent-ruby (1.1.10) + dnsruby (1.61.9) simpleidn (~> 0.1) - em-websocket (0.5.2) + em-websocket (0.5.3) eventmachine (>= 0.12.9) - http_parser.rb (~> 0.6.0) - ethon (0.14.0) + http_parser.rb (~> 0) + ethon (0.16.0) ffi (>= 1.15.0) eventmachine (1.2.7) execjs (2.8.1) - faraday (1.8.0) - faraday-em_http (~> 1.0) - faraday-em_synchrony (~> 1.0) - faraday-excon (~> 1.1) - faraday-httpclient (~> 1.0.1) - faraday-net_http (~> 1.0) - faraday-net_http_persistent (~> 1.1) - faraday-patron (~> 1.0) - faraday-rack (~> 1.0) - multipart-post (>= 1.2, < 3) + faraday (2.7.0) + faraday-net_http (>= 2.0, < 3.1) ruby2_keywords (>= 0.0.4) - faraday-em_http (1.0.0) - faraday-em_synchrony (1.0.0) - faraday-excon (1.1.0) - faraday-httpclient (1.0.1) - faraday-net_http (1.0.1) - faraday-net_http_persistent (1.2.0) - faraday-patron (1.0.0) - faraday-rack (1.0.0) - ffi (1.15.4) + faraday-net_http (3.0.2) + ffi (1.15.5) forwardable-extended (2.6.0) gemoji (3.0.1) - github-pages (209) - github-pages-health-check (= 1.16.1) - jekyll (= 3.9.0) + github-pages (227) + github-pages-health-check (= 1.17.9) + jekyll (= 3.9.2) jekyll-avatar (= 0.7.0) jekyll-coffeescript (= 1.1.1) - jekyll-commonmark-ghpages (= 0.1.6) + jekyll-commonmark-ghpages (= 0.2.0) jekyll-default-layout (= 0.1.4) jekyll-feed (= 0.15.1) jekyll-gist (= 1.5.0) jekyll-github-metadata (= 2.13.0) + jekyll-include-cache (= 0.2.1) jekyll-mentions (= 1.6.0) jekyll-optional-front-matter (= 0.3.2) jekyll-paginate (= 1.1.0) jekyll-readme-index (= 0.3.0) jekyll-redirect-from (= 0.16.0) jekyll-relative-links (= 0.6.1) - jekyll-remote-theme (= 0.4.2) + jekyll-remote-theme (= 0.4.3) jekyll-sass-converter (= 1.5.2) - jekyll-seo-tag (= 2.6.1) + jekyll-seo-tag (= 2.8.0) jekyll-sitemap (= 1.4.0) jekyll-swiss (= 1.0.0) - jekyll-theme-architect (= 0.1.1) - jekyll-theme-cayman (= 0.1.1) - jekyll-theme-dinky (= 0.1.1) - jekyll-theme-hacker (= 0.1.2) - jekyll-theme-leap-day (= 0.1.1) - jekyll-theme-merlot (= 0.1.1) - jekyll-theme-midnight (= 0.1.1) - jekyll-theme-minimal (= 0.1.1) - jekyll-theme-modernist (= 0.1.1) - jekyll-theme-primer (= 0.5.4) - jekyll-theme-slate (= 0.1.1) - jekyll-theme-tactile (= 0.1.1) - jekyll-theme-time-machine (= 0.1.1) + jekyll-theme-architect (= 0.2.0) + jekyll-theme-cayman (= 0.2.0) + jekyll-theme-dinky (= 0.2.0) + jekyll-theme-hacker (= 0.2.0) + jekyll-theme-leap-day (= 0.2.0) + jekyll-theme-merlot (= 0.2.0) + jekyll-theme-midnight (= 0.2.0) + jekyll-theme-minimal (= 0.2.0) + jekyll-theme-modernist (= 0.2.0) + jekyll-theme-primer (= 0.6.0) + jekyll-theme-slate (= 0.2.0) + jekyll-theme-tactile (= 0.2.0) + jekyll-theme-time-machine (= 0.2.0) jekyll-titles-from-headings (= 0.5.3) jemoji (= 0.12.0) - kramdown (= 2.3.0) + kramdown (= 2.3.2) kramdown-parser-gfm (= 1.1.0) liquid (= 4.0.3) mercenary (~> 0.3) minima (= 2.5.1) - nokogiri (>= 1.10.4, < 2.0) - rouge (= 3.23.0) + nokogiri (>= 1.13.6, < 2.0) + rouge (= 3.26.0) terminal-table (~> 1.4) - github-pages-health-check (1.16.1) + github-pages-health-check (1.17.9) addressable (~> 2.3) dnsruby (~> 1.60) octokit (~> 4.0) - public_suffix (~> 3.0) + public_suffix (>= 3.0, < 5.0) typhoeus (~> 1.3) - html-pipeline (2.14.0) + html-pipeline (2.14.3) activesupport (>= 2) nokogiri (>= 1.4) - http_parser.rb (0.6.0) + http_parser.rb (0.8.0) i18n (0.9.5) concurrent-ruby (~> 1.0) - jekyll (3.9.0) + jekyll (3.9.2) addressable (~> 2.4) colorator (~> 1.0) em-websocket (~> 0.5) @@ -122,12 +107,12 @@ GEM jekyll-coffeescript (1.1.1) coffee-script (~> 2.2) coffee-script-source (~> 1.11.1) - jekyll-commonmark (1.3.1) - commonmarker (~> 0.14) - jekyll (>= 3.7, < 5.0) - jekyll-commonmark-ghpages (0.1.6) - commonmarker (~> 0.17.6) - jekyll-commonmark (~> 1.2) + jekyll-commonmark (1.4.0) + commonmarker (~> 0.22) + jekyll-commonmark-ghpages (0.2.0) + commonmarker (~> 0.23.4) + jekyll (~> 3.9.0) + jekyll-commonmark (~> 1.4.0) rouge (>= 2.0, < 4.0) jekyll-default-layout (0.1.4) jekyll (~> 3.0) @@ -138,6 +123,8 @@ GEM jekyll-github-metadata (2.13.0) jekyll (>= 3.4, < 5.0) octokit (~> 4.0, != 4.4.0) + jekyll-include-cache (0.2.1) + jekyll (>= 3.7, < 5.0) jekyll-mentions (1.6.0) html-pipeline (~> 2.3) jekyll (>= 3.7, < 5.0) @@ -150,57 +137,57 @@ GEM jekyll (>= 3.3, < 5.0) jekyll-relative-links (0.6.1) jekyll (>= 3.3, < 5.0) - jekyll-remote-theme (0.4.2) + jekyll-remote-theme (0.4.3) addressable (~> 2.0) jekyll (>= 3.5, < 5.0) jekyll-sass-converter (>= 1.0, <= 3.0.0, != 2.0.0) rubyzip (>= 1.3.0, < 3.0) jekyll-sass-converter (1.5.2) sass (~> 3.4) - jekyll-seo-tag (2.6.1) - jekyll (>= 3.3, < 5.0) + jekyll-seo-tag (2.8.0) + jekyll (>= 3.8, < 5.0) jekyll-sitemap (1.4.0) jekyll (>= 3.7, < 5.0) jekyll-swiss (1.0.0) - jekyll-theme-architect (0.1.1) - jekyll (~> 3.5) + jekyll-theme-architect (0.2.0) + jekyll (> 3.5, < 5.0) jekyll-seo-tag (~> 2.0) - jekyll-theme-cayman (0.1.1) - jekyll (~> 3.5) + jekyll-theme-cayman (0.2.0) + jekyll (> 3.5, < 5.0) jekyll-seo-tag (~> 2.0) - jekyll-theme-dinky (0.1.1) - jekyll (~> 3.5) + jekyll-theme-dinky (0.2.0) + jekyll (> 3.5, < 5.0) jekyll-seo-tag (~> 2.0) - jekyll-theme-hacker (0.1.2) + jekyll-theme-hacker (0.2.0) jekyll (> 3.5, < 5.0) jekyll-seo-tag (~> 2.0) - jekyll-theme-leap-day (0.1.1) - jekyll (~> 3.5) + jekyll-theme-leap-day (0.2.0) + jekyll (> 3.5, < 5.0) jekyll-seo-tag (~> 2.0) - jekyll-theme-merlot (0.1.1) - jekyll (~> 3.5) + jekyll-theme-merlot (0.2.0) + jekyll (> 3.5, < 5.0) jekyll-seo-tag (~> 2.0) - jekyll-theme-midnight (0.1.1) - jekyll (~> 3.5) + jekyll-theme-midnight (0.2.0) + jekyll (> 3.5, < 5.0) jekyll-seo-tag (~> 2.0) - jekyll-theme-minimal (0.1.1) - jekyll (~> 3.5) + jekyll-theme-minimal (0.2.0) + jekyll (> 3.5, < 5.0) jekyll-seo-tag (~> 2.0) - jekyll-theme-modernist (0.1.1) - jekyll (~> 3.5) + jekyll-theme-modernist (0.2.0) + jekyll (> 3.5, < 5.0) jekyll-seo-tag (~> 2.0) - jekyll-theme-primer (0.5.4) + jekyll-theme-primer (0.6.0) jekyll (> 3.5, < 5.0) jekyll-github-metadata (~> 2.9) jekyll-seo-tag (~> 2.0) - jekyll-theme-slate (0.1.1) - jekyll (~> 3.5) + jekyll-theme-slate (0.2.0) + jekyll (> 3.5, < 5.0) jekyll-seo-tag (~> 2.0) - jekyll-theme-tactile (0.1.1) - jekyll (~> 3.5) + jekyll-theme-tactile (0.2.0) + jekyll (> 3.5, < 5.0) jekyll-seo-tag (~> 2.0) - jekyll-theme-time-machine (0.1.1) - jekyll (~> 3.5) + jekyll-theme-time-machine (0.2.0) + jekyll (> 3.5, < 5.0) jekyll-seo-tag (~> 2.0) jekyll-titles-from-headings (0.5.3) jekyll (>= 3.3, < 5.0) @@ -210,39 +197,36 @@ GEM gemoji (~> 3.0) html-pipeline (~> 2.2) jekyll (>= 3.0, < 5.0) - kramdown (2.3.0) + kramdown (2.3.2) rexml kramdown-parser-gfm (1.1.0) kramdown (~> 2.0) liquid (4.0.3) - listen (3.7.0) + listen (3.7.1) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) mercenary (0.3.6) - mini_portile2 (2.6.1) + mini_portile2 (2.8.0) minima (2.5.1) jekyll (>= 3.5, < 5.0) jekyll-feed (~> 0.9) jekyll-seo-tag (~> 2.1) - minitest (5.14.4) - multipart-post (2.1.1) - nokogiri (1.12.5) - mini_portile2 (~> 2.6.1) + minitest (5.16.3) + nokogiri (1.13.9) + mini_portile2 (~> 2.8.0) racc (~> 1.4) - octokit (4.21.0) - faraday (>= 0.9) - sawyer (~> 0.8.0, >= 0.5.3) + octokit (4.25.1) + faraday (>= 1, < 3) + sawyer (~> 0.9) pathutil (0.16.2) forwardable-extended (~> 2.6) - public_suffix (3.1.1) - racc (1.5.2) - rb-fsevent (0.11.0) + public_suffix (4.0.7) + racc (1.6.0) + rb-fsevent (0.11.2) rb-inotify (0.10.1) ffi (~> 1.0) rexml (3.2.5) - rouge (3.23.0) - ruby-enum (0.9.0) - i18n + rouge (3.26.0) ruby2_keywords (0.0.5) rubyzip (2.3.2) safe_yaml (1.0.5) @@ -251,9 +235,9 @@ GEM sass-listen (4.0.0) rb-fsevent (~> 0.9, >= 0.9.4) rb-inotify (~> 0.9, >= 0.9.7) - sawyer (0.8.2) + sawyer (0.9.2) addressable (>= 2.3.5) - faraday (> 0.8, < 2.0) + faraday (>= 0.17.3, < 3) simpleidn (0.2.1) unf (~> 0.1.4) terminal-table (1.8.0) @@ -261,22 +245,22 @@ GEM thread_safe (0.3.6) typhoeus (1.4.0) ethon (>= 0.9.0) - tzinfo (1.2.9) + tzinfo (1.2.10) thread_safe (~> 0.1) - tzinfo-data (1.2021.3) + tzinfo-data (1.2022.6) tzinfo (>= 1.0.0) unf (0.1.4) unf_ext - unf_ext (0.0.8) + unf_ext (0.0.8.2) unicode-display_width (1.8.0) wdm (0.1.1) - zeitwerk (2.4.2) + zeitwerk (2.6.6) PLATFORMS ruby DEPENDENCIES - github-pages (~> 209) + github-pages (~> 227) jekyll (~> 3.9.0) tzinfo (~> 1.2) tzinfo-data diff --git a/docs/_config.yml b/docs/_config.yml index 13b044a6d6..58aa4d08b6 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -27,6 +27,8 @@ kramdown: toc_levels: 1..3 remote_theme: rundocs/jekyll-rtd-theme@v2.0.10 +plugins: + - jekyll-remote-theme # Exclude from processing. # The following items will not be processed, by default. Create a custom list From a32ed5ecb1a6c872ba1a2e4177842a6c4dd116bc Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Thu, 17 Nov 2022 13:53:41 +0200 Subject: [PATCH 03/87] test/e2e: fix topologu-updater cmdline args The -kubelet-config-file flag got renamed to -kubelet-config-uri and the expected format changed from plain filepath to URI. --- test/e2e/utils/pod.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/utils/pod.go b/test/e2e/utils/pod.go index d156c9bba1..8726f41df3 100644 --- a/test/e2e/utils/pod.go +++ b/test/e2e/utils/pod.go @@ -311,7 +311,7 @@ func nfdTopologyUpdaterPodSpec(kc KubeletConfig, image string, extraArgs []strin ImagePullPolicy: pullPolicy(), Command: []string{"nfd-topology-updater"}, Args: append([]string{ - "--kubelet-config-file=/podresources/config.yaml", + "--kubelet-config-uri=file:///podresources/config.yaml", "--podresources-socket=unix:///podresources/kubelet.sock", "--sleep-interval=3s", "--watch-namespace=rte", From a65278d890adcd0d46287579f86c8143d85e5286 Mon Sep 17 00:00:00 2001 From: Talor Itzhak Date: Thu, 17 Nov 2022 16:57:20 +0200 Subject: [PATCH 04/87] e2e: podSecurity: adapt PodSecurity constraints The tested pods have some lax spec wrt security, hence a restrict podSecurity namespace won't allow running those pods. In topology-updater tests, the topology-updater pod needs to run the container as root so change the namespace podSecurity from restricted to priviliged. In node-feature-discovery tests, we don't need root access, so add the required security context configuration. Signed-off-by: Talor Itzhak --- test/e2e/node_feature_discovery.go | 3 +-- test/e2e/topology_updater.go | 3 ++- test/e2e/utils/pod.go | 11 +++++++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index 141215445f..f166ca3e7b 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -38,9 +38,8 @@ import ( e2elog "k8s.io/kubernetes/test/e2e/framework/log" e2enetwork "k8s.io/kubernetes/test/e2e/framework/network" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" - nfdclient "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned" - nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/pkg/apis/nfd/v1alpha1" + nfdclient "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned" "sigs.k8s.io/node-feature-discovery/source/custom" testutils "sigs.k8s.io/node-feature-discovery/test/e2e/utils" ) diff --git a/test/e2e/topology_updater.go b/test/e2e/topology_updater.go index 266625be2c..42afbdd287 100644 --- a/test/e2e/topology_updater.go +++ b/test/e2e/topology_updater.go @@ -36,6 +36,7 @@ import ( "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework/kubelet" e2enetwork "k8s.io/kubernetes/test/e2e/framework/network" + admissionapi "k8s.io/pod-security-admission/api" testutils "sigs.k8s.io/node-feature-discovery/test/e2e/utils" ) @@ -51,7 +52,7 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { ) f := framework.NewDefaultFramework("node-topology-updater") - + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged BeforeEach(func() { var err error diff --git a/test/e2e/utils/pod.go b/test/e2e/utils/pod.go index 8726f41df3..fd6c0c43e8 100644 --- a/test/e2e/utils/pod.go +++ b/test/e2e/utils/pod.go @@ -203,6 +203,8 @@ func newDaemonSet(name string, podSpec *corev1.PodSpec) *appsv1.DaemonSet { } func nfdWorkerPodSpec(image string, extraArgs []string) *corev1.PodSpec { + yes := true + no := false return &corev1.PodSpec{ Containers: []corev1.Container{ { @@ -221,6 +223,15 @@ func nfdWorkerPodSpec(image string, extraArgs []string) *corev1.PodSpec { }, }, }, + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + Privileged: &no, + RunAsNonRoot: &yes, + ReadOnlyRootFilesystem: &yes, + AllowPrivilegeEscalation: &no, + }, VolumeMounts: []corev1.VolumeMount{ { Name: "host-boot", From 1b6fad6427803482e644eb0384d7d30760605002 Mon Sep 17 00:00:00 2001 From: Talor Itzhak Date: Thu, 17 Nov 2022 17:47:40 +0200 Subject: [PATCH 05/87] e2e: topologyupdater: insert condition check into eventually We might not get the most updated node topology resource on the first `GET` call. Hence, put the whole check inside `Eventually`, and check for the most updated node topology resource on every iteration. Signed-off-by: Talor Itzhak --- test/e2e/topology_updater.go | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/test/e2e/topology_updater.go b/test/e2e/topology_updater.go index 42afbdd287..2eb047b39a 100644 --- a/test/e2e/topology_updater.go +++ b/test/e2e/topology_updater.go @@ -210,13 +210,17 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { initialNodeTopo := testutils.GetNodeTopology(topologyClient, topologyUpdaterNode.Name) By("creating a pod consuming exclusive CPUs") sleeperPod := testutils.GuaranteedSleeperPod("1000m") + // in case there is more than a single node in the cluster + // we need to set the node name, so we'll have certainty about + // which node we need to examine + sleeperPod.Spec.NodeName = topologyUpdaterNode.Name podMap := make(map[string]*corev1.Pod) pod := f.PodClient().CreateSync(sleeperPod) podMap[pod.Name] = pod defer testutils.DeletePodsAsync(f, podMap) - By("getting the updated topology") + By("checking the changes in the updated topology") var finalNodeTopo *v1alpha1.NodeResourceTopology Eventually(func() bool { finalNodeTopo, err = topologyClient.TopologyV1alpha1().NodeResourceTopologies().Get(context.TODO(), topologyUpdaterNode.Name, metav1.GetOptions{}) @@ -224,18 +228,23 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { framework.Logf("failed to get the node topology resource: %v", err) return false } - return finalNodeTopo.ObjectMeta.ResourceVersion != initialNodeTopo.ObjectMeta.ResourceVersion - }, time.Minute, 5*time.Second).Should(BeTrue(), "didn't get updated node topology info") - By("checking the changes in the updated topology") + if finalNodeTopo.ObjectMeta.ResourceVersion == initialNodeTopo.ObjectMeta.ResourceVersion { + framework.Logf("node topology resource %s was not updated", topologyUpdaterNode.Name) + } - initialAllocRes := testutils.AllocatableResourceListFromNodeResourceTopology(initialNodeTopo) - finalAllocRes := testutils.AllocatableResourceListFromNodeResourceTopology(finalNodeTopo) - if len(initialAllocRes) == 0 || len(finalAllocRes) == 0 { - Fail(fmt.Sprintf("failed to find allocatable resources from node topology initial=%v final=%v", initialAllocRes, finalAllocRes)) - } - zoneName, resName, isLess := lessAllocatableResources(initialAllocRes, finalAllocRes) - framework.Logf("zone=%q resource=%q isLess=%v", zoneName, resName, isLess) - Expect(isLess).To(BeTrue(), fmt.Sprintf("final allocatable resources not decreased - initial=%v final=%v", initialAllocRes, finalAllocRes)) + initialAllocRes := testutils.AllocatableResourceListFromNodeResourceTopology(initialNodeTopo) + finalAllocRes := testutils.AllocatableResourceListFromNodeResourceTopology(finalNodeTopo) + if len(initialAllocRes) == 0 || len(finalAllocRes) == 0 { + Fail(fmt.Sprintf("failed to find allocatable resources from node topology initial=%v final=%v", initialAllocRes, finalAllocRes)) + } + + zoneName, resName, isLess := lessAllocatableResources(initialAllocRes, finalAllocRes) + framework.Logf("zone=%q resource=%q isLess=%v", zoneName, resName, isLess) + if !isLess { + framework.Logf("final allocatable resources not decreased - initial=%v final=%v", initialAllocRes, finalAllocRes) + } + return true + }, time.Minute, 5*time.Second).Should(BeTrue(), "didn't get updated node topology info") }) }) From 546d0c3a1bfa0760739af311301f904e5d5f0575 Mon Sep 17 00:00:00 2001 From: Talor Itzhak Date: Thu, 17 Nov 2022 20:23:31 +0200 Subject: [PATCH 06/87] e2e: topologyupdater: wait for deletion It might take time for the CRD to get deleted and it might cause some falkiness in the tests. Now before we create the CRD, we make sure to delete the old object, wait for it deletion to complete and only then create a new CRD object. Signed-off-by: Talor Itzhak --- test/e2e/topology_updater.go | 9 +-------- test/e2e/utils/noderesourcetopology.go | 22 +++++++++++++++++----- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/test/e2e/topology_updater.go b/test/e2e/topology_updater.go index 2eb047b39a..0d8a435012 100644 --- a/test/e2e/topology_updater.go +++ b/test/e2e/topology_updater.go @@ -28,7 +28,6 @@ import ( topologyclientset "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/generated/clientset/versioned" corev1 "k8s.io/api/core/v1" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" extclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -45,7 +44,6 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { var ( extClient *extclient.Clientset topologyClient *topologyclientset.Clientset - crd *apiextensionsv1.CustomResourceDefinition topologyUpdaterNode *corev1.Node workerNodes []corev1.Node kubeletConfig *kubeletconfig.KubeletConfiguration @@ -73,7 +71,7 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { By(fmt.Sprintf("Using config (%#v)", kcfg)) By("Creating the node resource topologies CRD") - crd, err = testutils.CreateNodeResourceTopologies(extClient) + _, err = testutils.CreateNodeResourceTopologies(extClient) Expect(err).NotTo(HaveOccurred()) err = testutils.ConfigureRBAC(f.ClientSet, f.Namespace.Name) @@ -254,11 +252,6 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { if err != nil { framework.Logf("failed to delete RBAC resources: %v", err) } - - err = extClient.ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), crd.Name, metav1.DeleteOptions{}) - if err != nil { - framework.Logf("failed to delete node resources topologies CRD: %v", err) - } }) }) diff --git a/test/e2e/utils/noderesourcetopology.go b/test/e2e/utils/noderesourcetopology.go index 7ff6988e6d..a685680060 100644 --- a/test/e2e/utils/noderesourcetopology.go +++ b/test/e2e/utils/noderesourcetopology.go @@ -35,6 +35,7 @@ import ( extclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes/scheme" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" "k8s.io/kubernetes/test/e2e/framework" @@ -81,15 +82,26 @@ func CreateNodeResourceTopologies(extClient extclient.Interface) (*apiextensions return nil, err } - updatedCrd, err := extClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), crd.Name, metav1.GetOptions{}) + // Delete existing CRD (if any) with this we also get rid of stale objects + err = extClient.ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), crd.Name, metav1.DeleteOptions{}) if err != nil && !errors.IsNotFound(err) { - return nil, err + return nil, fmt.Errorf("failed to delete NodeResourceTopology CRD: %w", err) } - if err == nil { - return updatedCrd, nil - } + // It takes time for the delete operation, wait until the CRD completely gone + if err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) { + _, err = extClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), crd.Name, metav1.GetOptions{}) + if err == nil { + return false, nil + } + if errors.IsNotFound(err) { + return true, nil + } + return false, err + }); err != nil { + return nil, fmt.Errorf("failed to get NodeResourceTopology CRD: %w", err) + } return extClient.ApiextensionsV1().CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{}) } From 5b0788ced410c36689cf0348c7db4a96f5d2c86d Mon Sep 17 00:00:00 2001 From: Talor Itzhak Date: Wed, 2 Nov 2022 16:01:25 +0200 Subject: [PATCH 07/87] topology-updater: introduce exclude-list The exclude-list allows to filter specific resource accounting from NRT's objects per node basis. The CRs created by the topology-updater are used by the scheduler-plugin as a source of truth for making scheduling decisions. As such, this feature allows to hide specific information from the scheduler, which in turn will affect the scheduling decision. A common use case is when user would like to perform scheduling decisions which are based on a specific resource. In that case, we can exclude all the other resources which we don't want the scheduler to exemine. The exclude-list is provided to the topology-updater via a ConfigMap. Resource type's names specified in the list should match the names as shown here: https://pkg.go.dev/k8s.io/api/core/v1#ResourceName This is a resurrection of an old work started here: https://github.com/kubernetes-sigs/node-feature-discovery/pull/545 Signed-off-by: Talor Itzhak --- cmd/nfd-topology-updater/main.go | 2 + cmd/nfd-topology-updater/main_test.go | 7 +- .../topology-updater/nfd-topology-updater.go | 47 ++++++++++++- pkg/resourcemonitor/excludelist.go | 33 +++++++++ pkg/resourcemonitor/excludelist_test.go | 70 +++++++++++++++++++ .../noderesourcesaggregator.go | 13 ++-- .../noderesourcesaggregator_test.go | 4 +- 7 files changed, 166 insertions(+), 10 deletions(-) create mode 100644 pkg/resourcemonitor/excludelist.go create mode 100644 pkg/resourcemonitor/excludelist_test.go diff --git a/cmd/nfd-topology-updater/main.go b/cmd/nfd-topology-updater/main.go index bdb9b7c924..0b9bef9e80 100644 --- a/cmd/nfd-topology-updater/main.go +++ b/cmd/nfd-topology-updater/main.go @@ -155,6 +155,8 @@ func initFlags(flagset *flag.FlagSet) (*topology.Args, *resourcemonitor.Args) { "NFD server address to connecto to.") flagset.StringVar(&args.ServerNameOverride, "server-name-override", "", "Hostname expected from server certificate, useful in testing") + flagset.StringVar(&args.ConfigFile, "config", "/etc/kubernetes/node-feature-discovery/nfd-topology-updater.conf", + "Config file to use.") klog.InitFlags(flagset) diff --git a/cmd/nfd-topology-updater/main_test.go b/cmd/nfd-topology-updater/main_test.go index e1b249808a..dd2ba86e08 100644 --- a/cmd/nfd-topology-updater/main_test.go +++ b/cmd/nfd-topology-updater/main_test.go @@ -34,19 +34,22 @@ func TestArgsParse(t *testing.T) { Convey("noPublish is set and args.sources is set to the default value", func() { So(args.NoPublish, ShouldBeTrue) So(args.Oneshot, ShouldBeTrue) + So(args.ConfigFile, ShouldEqual, "/etc/kubernetes/node-feature-discovery/nfd-topology-updater.conf") So(finderArgs.SleepInterval, ShouldEqual, 60*time.Second) So(finderArgs.PodResourceSocketPath, ShouldEqual, "/var/lib/kubelet/pod-resources/kubelet.sock") }) }) - Convey("When valid args are specified for -kubelet-config-url and -sleep-interval,", func() { + Convey("When valid args are specified for -kubelet-config-url, -sleep-interval and -config,", func() { args, finderArgs := parseArgs(flags, "-kubelet-config-uri=file:///path/testconfig.yaml", - "-sleep-interval=30s") + "-sleep-interval=30s", + "-config=/path/nfd-topology-updater.conf") Convey("args.sources is set to appropriate values", func() { So(args.NoPublish, ShouldBeFalse) So(args.Oneshot, ShouldBeFalse) + So(args.ConfigFile, ShouldEqual, "/path/nfd-topology-updater.conf") So(finderArgs.SleepInterval, ShouldEqual, 30*time.Second) So(finderArgs.KubeletConfigURI, ShouldEqual, "file:///path/testconfig.yaml") So(finderArgs.PodResourceSocketPath, ShouldEqual, "/var/lib/kubelet/pod-resources/kubelet.sock") diff --git a/pkg/nfd-client/topology-updater/nfd-topology-updater.go b/pkg/nfd-client/topology-updater/nfd-topology-updater.go index baa8f378e6..1853536e20 100644 --- a/pkg/nfd-client/topology-updater/nfd-topology-updater.go +++ b/pkg/nfd-client/topology-updater/nfd-topology-updater.go @@ -18,6 +18,8 @@ package topologyupdater import ( "fmt" + "os" + "path/filepath" "time" "k8s.io/klog/v2" @@ -32,6 +34,7 @@ import ( pb "sigs.k8s.io/node-feature-discovery/pkg/topologyupdater" "sigs.k8s.io/node-feature-discovery/pkg/utils" "sigs.k8s.io/node-feature-discovery/pkg/version" + "sigs.k8s.io/yaml" ) // Args are the command line arguments @@ -40,6 +43,12 @@ type Args struct { NoPublish bool Oneshot bool KubeConfigFile string + ConfigFile string +} + +// NFDConfig contains the configuration settings of NFDTopologyUpdater. +type NFDConfig struct { + ExcludeList map[string][]string } type NfdTopologyUpdater interface { @@ -59,6 +68,8 @@ type nfdTopologyUpdater struct { certWatch *utils.FsWatcher client pb.NodeTopologyClient stop chan struct{} // channel for signaling stop + configFilePath string + config *NFDConfig } // NewTopologyUpdater creates a new NfdTopologyUpdater instance. @@ -75,7 +86,11 @@ func NewTopologyUpdater(args Args, resourcemonitorArgs resourcemonitor.Args, pol nodeInfo: &staticNodeInfo{ tmPolicy: policy, }, - stop: make(chan struct{}, 1), + stop: make(chan struct{}, 1), + config: &NFDConfig{}, + } + if args.ConfigFile != "" { + nfd.configFilePath = filepath.Clean(args.ConfigFile) } return nfd, nil } @@ -99,6 +114,9 @@ func (w *nfdTopologyUpdater) Run() error { } kubeApihelper = apihelper.K8sHelpers{Kubeconfig: kubeconfig} } + if err := w.configure(); err != nil { + return fmt.Errorf("faild to configure Node Feature Discovery Topology Updater: %w", err) + } var resScan resourcemonitor.ResourcesScanner @@ -113,7 +131,8 @@ func (w *nfdTopologyUpdater) Run() error { // zonesChannel := make(chan v1alpha1.ZoneList) var zones v1alpha1.ZoneList - resAggr, err := resourcemonitor.NewResourcesAggregator(podResClient) + excludeList := resourcemonitor.NewExcludeResourceList(w.config.ExcludeList, nfdclient.NodeName()) + resAggr, err := resourcemonitor.NewResourcesAggregator(podResClient, excludeList) if err != nil { return fmt.Errorf("failed to obtain node resource information: %w", err) } @@ -245,3 +264,27 @@ func advertiseNodeTopology(client pb.NodeTopologyClient, zoneInfo v1alpha1.ZoneL return nil } + +func (w *nfdTopologyUpdater) configure() error { + if w.configFilePath == "" { + klog.Warningf("file path for nfd-topology-updater conf file is empty") + return nil + } + + b, err := os.ReadFile(w.configFilePath) + if err != nil { + // config is optional + if os.IsNotExist(err) { + klog.Warningf("couldn't find conf file under %v", w.configFilePath) + return nil + } + return err + } + + err = yaml.Unmarshal(b, w.config) + if err != nil { + return fmt.Errorf("failed to parse configuration file %q: %w", w.configFilePath, err) + } + klog.Infof("configuration file %q parsed:\n %v", w.configFilePath, w.config) + return nil +} diff --git a/pkg/resourcemonitor/excludelist.go b/pkg/resourcemonitor/excludelist.go new file mode 100644 index 0000000000..444042f469 --- /dev/null +++ b/pkg/resourcemonitor/excludelist.go @@ -0,0 +1,33 @@ +package resourcemonitor + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" +) + +// ExcludeResourceList contains a list of resources to ignore during resources scan +type ExcludeResourceList struct { + excludeList sets.String +} + +// NewExcludeResourceList returns new ExcludeList with values with set.String types +func NewExcludeResourceList(resMap map[string][]string, nodeName string) ExcludeResourceList { + excludeList := make(sets.String) + for k, v := range resMap { + if k == nodeName || k == "*" { + excludeList.Insert(v...) + } + } + return ExcludeResourceList{ + excludeList: excludeList, + } +} + +func (rl *ExcludeResourceList) IsExcluded(resource corev1.ResourceName) bool { + if rl.excludeList.Has(string(resource)) { + klog.V(5).InfoS("resource excluded", "resource", resource) + return true + } + return false +} diff --git a/pkg/resourcemonitor/excludelist_test.go b/pkg/resourcemonitor/excludelist_test.go new file mode 100644 index 0000000000..092d1da42c --- /dev/null +++ b/pkg/resourcemonitor/excludelist_test.go @@ -0,0 +1,70 @@ +package resourcemonitor + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" +) + +const ( + cpu = string(corev1.ResourceCPU) + memory = string(corev1.ResourceMemory) + hugepages2Mi = "hugepages-2Mi" + nicResourceName = "vendor/nic1" +) + +func TestNewExcludeResourceList(t *testing.T) { + tests := []struct { + desc string + excludeListConfig map[string][]string + nodeName string + expectedExcludedResources []string + }{ + { + + desc: "exclude list with multiple nodes", + excludeListConfig: map[string][]string{ + "node1": { + cpu, + nicResourceName, + }, + "node2": { + memory, + hugepages2Mi, + }, + }, + nodeName: "node1", + expectedExcludedResources: []string{cpu, nicResourceName}, + }, + { + desc: "exclude list with wild card", + excludeListConfig: map[string][]string{ + "*": { + memory, nicResourceName, + }, + "node1": { + cpu, + hugepages2Mi, + }, + }, + nodeName: "node2", + expectedExcludedResources: []string{memory, nicResourceName}, + }, + { + desc: "empty exclude list", + excludeListConfig: map[string][]string{}, + nodeName: "node1", + expectedExcludedResources: []string{}, + }, + } + + for _, tt := range tests { + t.Logf("test %s", tt.desc) + excludeList := NewExcludeResourceList(tt.excludeListConfig, tt.nodeName) + for _, res := range tt.expectedExcludedResources { + if !excludeList.IsExcluded(corev1.ResourceName(res)) { + t.Errorf("resource: %q expected to be excluded from node: %q", res, tt.nodeName) + } + } + } +} diff --git a/pkg/resourcemonitor/noderesourcesaggregator.go b/pkg/resourcemonitor/noderesourcesaggregator.go index 8b08c33aea..12d4676f21 100644 --- a/pkg/resourcemonitor/noderesourcesaggregator.go +++ b/pkg/resourcemonitor/noderesourcesaggregator.go @@ -28,8 +28,8 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/klog/v2" - podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1" + podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1" "sigs.k8s.io/node-feature-discovery/pkg/utils" "sigs.k8s.io/node-feature-discovery/pkg/utils/hostpath" ) @@ -46,6 +46,7 @@ type nodeResources struct { topo *ghw.TopologyInfo reservedCPUIDPerNUMA map[int][]string memoryResourcesCapacityPerNUMA utils.NumaMemoryResources + excludeList ExcludeResourceList } type resourceData struct { @@ -54,7 +55,7 @@ type resourceData struct { capacity int64 } -func NewResourcesAggregator(podResourceClient podresourcesapi.PodResourcesListerClient) (ResourcesAggregator, error) { +func NewResourcesAggregator(podResourceClient podresourcesapi.PodResourcesListerClient, excludeList ExcludeResourceList) (ResourcesAggregator, error) { var err error topo, err := ghw.Topology(ghw.WithPathOverrides(ghw.PathOverrides{ @@ -85,11 +86,11 @@ func NewResourcesAggregator(podResourceClient podresourcesapi.PodResourcesLister return nil, fmt.Errorf("failed to get allocatable resources (ensure that KubeletPodResourcesGetAllocatable feature gate is enabled): %w", err) } - return NewResourcesAggregatorFromData(topo, resp, memoryResourcesCapacityPerNUMA), nil + return NewResourcesAggregatorFromData(topo, resp, memoryResourcesCapacityPerNUMA, excludeList), nil } // NewResourcesAggregatorFromData is used to aggregate resource information based on the received data from underlying hardware and podresource API -func NewResourcesAggregatorFromData(topo *ghw.TopologyInfo, resp *podresourcesapi.AllocatableResourcesResponse, memoryResourceCapacity utils.NumaMemoryResources) ResourcesAggregator { +func NewResourcesAggregatorFromData(topo *ghw.TopologyInfo, resp *podresourcesapi.AllocatableResourcesResponse, memoryResourceCapacity utils.NumaMemoryResources, excludeList ExcludeResourceList) ResourcesAggregator { allDevs := getContainerDevicesFromAllocatableResources(resp, topo) return &nodeResources{ topo: topo, @@ -97,6 +98,7 @@ func NewResourcesAggregatorFromData(topo *ghw.TopologyInfo, resp *podresourcesap perNUMAAllocatable: makeNodeAllocatable(allDevs, resp.GetMemory()), reservedCPUIDPerNUMA: makeReservedCPUMap(topo.Nodes, allDevs), memoryResourcesCapacityPerNUMA: memoryResourceCapacity, + excludeList: excludeList, } } @@ -108,6 +110,9 @@ func (noderesourceData *nodeResources) Aggregate(podResData []PodResources) topo if ok { perNuma[nodeID] = make(map[corev1.ResourceName]*resourceData) for resName, allocatable := range nodeRes { + if noderesourceData.excludeList.IsExcluded(resName) { + continue + } switch { case resName == "cpu": perNuma[nodeID][resName] = &resourceData{ diff --git a/pkg/resourcemonitor/noderesourcesaggregator_test.go b/pkg/resourcemonitor/noderesourcesaggregator_test.go index a5ddeca89c..011c14de90 100644 --- a/pkg/resourcemonitor/noderesourcesaggregator_test.go +++ b/pkg/resourcemonitor/noderesourcesaggregator_test.go @@ -178,7 +178,7 @@ func TestResourcesAggregator(t *testing.T) { corev1.ResourceName("hugepages-2Mi"): 2048, }, } - resAggr = NewResourcesAggregatorFromData(&fakeTopo, availRes, memoryResourcesCapacity) + resAggr = NewResourcesAggregatorFromData(&fakeTopo, availRes, memoryResourcesCapacity, NewExcludeResourceList(map[string][]string{}, "")) Convey("When aggregating resources", func() { expected := topologyv1alpha1.ZoneList{ @@ -376,7 +376,7 @@ func TestResourcesAggregator(t *testing.T) { }, } - resAggr = NewResourcesAggregatorFromData(&fakeTopo, availRes, memoryResourcesCapacity) + resAggr = NewResourcesAggregatorFromData(&fakeTopo, availRes, memoryResourcesCapacity, NewExcludeResourceList(map[string][]string{}, "")) Convey("When aggregating resources", func() { podRes := []PodResources{ From 1385d502222f671e53c8bc06c487409260c78609 Mon Sep 17 00:00:00 2001 From: Talor Itzhak Date: Sun, 20 Nov 2022 17:18:17 +0200 Subject: [PATCH 08/87] e2e: topology-updater: decouple configuration from creation Different tests requires different configuration of the topology-updater DaemonSet. Here, we decouple the configuration from the creation part using `JustBeforeEach` so that each test container will has its own configuration. Additional reading: https://onsi.github.io/ginkgo/#separating-creation-and-configuration-justbeforeeach Signed-off-by: Talor Itzhak --- test/e2e/topology_updater.go | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/test/e2e/topology_updater.go b/test/e2e/topology_updater.go index 0d8a435012..9b1a358769 100644 --- a/test/e2e/topology_updater.go +++ b/test/e2e/topology_updater.go @@ -27,6 +27,7 @@ import ( "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/apis/topology/v1alpha1" topologyclientset "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/generated/clientset/versioned" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" extclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -42,16 +43,17 @@ import ( var _ = SIGDescribe("Node Feature Discovery topology updater", func() { var ( - extClient *extclient.Clientset - topologyClient *topologyclientset.Clientset - topologyUpdaterNode *corev1.Node - workerNodes []corev1.Node - kubeletConfig *kubeletconfig.KubeletConfiguration + extClient *extclient.Clientset + topologyClient *topologyclientset.Clientset + topologyUpdaterNode *corev1.Node + topologyUpdaterDaemonSet *appsv1.DaemonSet + workerNodes []corev1.Node + kubeletConfig *kubeletconfig.KubeletConfiguration ) f := framework.NewDefaultFramework("node-topology-updater") f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged - BeforeEach(func() { + JustBeforeEach(func() { var err error if extClient == nil { @@ -64,12 +66,6 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { Expect(err).NotTo(HaveOccurred()) } - cfg, err := testutils.GetConfig() - Expect(err).ToNot(HaveOccurred()) - - kcfg := cfg.GetKubeletConfig() - By(fmt.Sprintf("Using config (%#v)", kcfg)) - By("Creating the node resource topologies CRD") _, err = testutils.CreateNodeResourceTopologies(extClient) Expect(err).NotTo(HaveOccurred()) @@ -88,7 +84,6 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { Expect(e2enetwork.WaitForService(f.ClientSet, f.Namespace.Name, masterService.Name, true, time.Second, 10*time.Second)).NotTo(HaveOccurred()) By("Creating nfd-topology-updater daemonset") - topologyUpdaterDaemonSet := testutils.NFDTopologyUpdaterDaemonSet(kcfg, fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag), []string{}) topologyUpdaterDaemonSet, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), topologyUpdaterDaemonSet, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -111,6 +106,16 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { }) Context("with single nfd-master pod", func() { + BeforeEach(func() { + cfg, err := testutils.GetConfig() + Expect(err).ToNot(HaveOccurred()) + + kcfg := cfg.GetKubeletConfig() + By(fmt.Sprintf("Using config (%#v)", kcfg)) + + topologyUpdaterDaemonSet = testutils.NFDTopologyUpdaterDaemonSet(kcfg, fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag), []string{}) + }) + It("should fill the node resource topologies CR with the data", func() { nodeTopology := testutils.GetNodeTopology(topologyClient, topologyUpdaterNode.Name) isValid := testutils.IsValidNodeTopology(nodeTopology, kubeletConfig) From 2896541bf16fcc57fdc60a1dc38447fdc6a10cc3 Mon Sep 17 00:00:00 2001 From: Talor Itzhak Date: Sun, 20 Nov 2022 17:21:49 +0200 Subject: [PATCH 09/87] e2e: topology-updater: add e2e test for exclude-list feature Signed-off-by: Talor Itzhak --- test/e2e/topology_updater.go | 63 ++++++++++++++++++++++++++++++------ test/e2e/utils/pod.go | 28 +++++++++++++++- 2 files changed, 81 insertions(+), 10 deletions(-) diff --git a/test/e2e/topology_updater.go b/test/e2e/topology_updater.go index 9b1a358769..375c84fead 100644 --- a/test/e2e/topology_updater.go +++ b/test/e2e/topology_updater.go @@ -67,11 +67,9 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { } By("Creating the node resource topologies CRD") - _, err = testutils.CreateNodeResourceTopologies(extClient) - Expect(err).NotTo(HaveOccurred()) + Expect(testutils.CreateNodeResourceTopologies(extClient)).ToNot(BeNil()) - err = testutils.ConfigureRBAC(f.ClientSet, f.Namespace.Name) - Expect(err).NotTo(HaveOccurred()) + Expect(testutils.ConfigureRBAC(f.ClientSet, f.Namespace.Name)).NotTo(HaveOccurred()) image := fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag) f.PodClient().CreateSync(testutils.NFDMasterPod(image, false)) @@ -105,6 +103,14 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { Expect(err).NotTo(HaveOccurred()) }) + // TODO: replace with regular AfterEach once we have https://github.com/kubernetes/kubernetes/pull/111998 in + f.AddAfterEach("Node Feature Discovery topology updater CRD and RBAC removal", func(f *framework.Framework, failed bool) { + err := testutils.DeconfigureRBAC(f.ClientSet, f.Namespace.Name) + if err != nil { + framework.Logf("failed to delete RBAC resources: %v", err) + } + }) + Context("with single nfd-master pod", func() { BeforeEach(func() { cfg, err := testutils.GetConfig() @@ -252,11 +258,50 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { }) - JustAfterEach(func() { - err := testutils.DeconfigureRBAC(f.ClientSet, f.Namespace.Name) - if err != nil { - framework.Logf("failed to delete RBAC resources: %v", err) - } + When("topology-updater configure to exclude memory", func() { + var topologyUpdaterConfigMap *corev1.ConfigMap + + BeforeEach(func() { + data := make(map[string]string) + data["nfd-topology-updater.conf"] = `excludeList: + '*': [memory] +` + topologyUpdaterConfigMap = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nfd-topology-updater-conf", + }, + Data: data, + } + + cm, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(context.TODO(), topologyUpdaterConfigMap, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + + cfg, err := testutils.GetConfig() + Expect(err).ToNot(HaveOccurred()) + + kcfg := cfg.GetKubeletConfig() + By(fmt.Sprintf("Using config (%#v)", kcfg)) + + opts := testutils.SpecWithConfigMap(cm.Name, cm.Name, "/etc/kubernetes/node-feature-discovery") + topologyUpdaterDaemonSet = testutils.NFDTopologyUpdaterDaemonSet(kcfg, fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag), []string{}, opts) + }) + + It("noderesourcetopology should not advertise the memory resource", func() { + Eventually(func() bool { + memoryFound := false + nodeTopology := testutils.GetNodeTopology(topologyClient, topologyUpdaterNode.Name) + for _, zone := range nodeTopology.Zones { + for _, res := range zone.Resources { + if res.Name == string(corev1.ResourceMemory) { + memoryFound = true + framework.Logf("resource:%s was found for nodeTopology:%s on zone:%s while it should not", corev1.ResourceMemory, nodeTopology.Name, zone.Name) + break + } + } + } + return memoryFound + }, 1*time.Minute, 10*time.Second).Should(BeFalse()) + }) }) }) diff --git a/test/e2e/utils/pod.go b/test/e2e/utils/pod.go index fd6c0c43e8..f1db373b9b 100644 --- a/test/e2e/utils/pod.go +++ b/test/e2e/utils/pod.go @@ -176,11 +176,37 @@ func NFDWorkerDaemonSet(image string, extraArgs []string) *appsv1.DaemonSet { } // NFDTopologyUpdaterDaemonSet provides the NFD daemon set topology updater -func NFDTopologyUpdaterDaemonSet(kc KubeletConfig, image string, extraArgs []string) *appsv1.DaemonSet { +func NFDTopologyUpdaterDaemonSet(kc KubeletConfig, image string, extraArgs []string, options ...func(spec *corev1.PodSpec)) *appsv1.DaemonSet { podSpec := nfdTopologyUpdaterPodSpec(kc, image, extraArgs) + for _, o := range options { + o(podSpec) + } return newDaemonSet("nfd-topology-updater", podSpec) } +func SpecWithConfigMap(cmName, volumeName, mountPath string) func(spec *corev1.PodSpec) { + return func(spec *corev1.PodSpec) { + spec.Volumes = append(spec.Volumes, + corev1.Volume{ + Name: volumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cmName, + }, + }, + }, + }) + cnt := &spec.Containers[0] + cnt.VolumeMounts = append(cnt.VolumeMounts, + corev1.VolumeMount{ + Name: volumeName, + ReadOnly: true, + MountPath: mountPath, + }) + } +} + // newDaemonSet provide the new daemon set func newDaemonSet(name string, podSpec *corev1.PodSpec) *appsv1.DaemonSet { return &appsv1.DaemonSet{ From 8b5918a2e9edf8a6d7ae14a7c802d0b70e88bcc0 Mon Sep 17 00:00:00 2001 From: Talor Itzhak Date: Wed, 2 Nov 2022 17:01:55 +0200 Subject: [PATCH 10/87] kustomize: topology-updater: enable the configuration via kustomization Add a kustomization file with a config example for the exclude-list. Signed-off-by: Talor Itzhak --- .../topology-updater-config/kustomization.yaml | 10 ++++++++++ .../nfd-topology-updater.conf.example | 7 +++++++ .../topology-updater/topologyupdater-mounts.yaml | 6 ++++++ .../master-worker-topologyupdater/kustomization.yaml | 1 + deployment/overlays/topologyupdater/kustomization.yaml | 1 + 5 files changed, 25 insertions(+) create mode 100644 deployment/components/topology-updater-config/kustomization.yaml create mode 100644 deployment/components/topology-updater-config/nfd-topology-updater.conf.example diff --git a/deployment/components/topology-updater-config/kustomization.yaml b/deployment/components/topology-updater-config/kustomization.yaml new file mode 100644 index 0000000000..73644ab499 --- /dev/null +++ b/deployment/components/topology-updater-config/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +generatorOptions: + disableNameSuffixHash: true + +configMapGenerator: +- files: + - nfd-topology-updater.conf=nfd-topology-updater.conf.example + name: nfd-topology-updater-conf diff --git a/deployment/components/topology-updater-config/nfd-topology-updater.conf.example b/deployment/components/topology-updater-config/nfd-topology-updater.conf.example new file mode 100644 index 0000000000..c244ba788f --- /dev/null +++ b/deployment/components/topology-updater-config/nfd-topology-updater.conf.example @@ -0,0 +1,7 @@ +## key = node name, value = list of resources to be excluded. +## use * to exclude from all nodes. +## an example for how the exclude list should looks like +#excludeList: +# node1: [cpu] +# node2: [memory, example/deviceA] +# *: [hugepages-2Mi] diff --git a/deployment/components/topology-updater/topologyupdater-mounts.yaml b/deployment/components/topology-updater/topologyupdater-mounts.yaml index 89f1c2e5e2..a098edf6c8 100644 --- a/deployment/components/topology-updater/topologyupdater-mounts.yaml +++ b/deployment/components/topology-updater/topologyupdater-mounts.yaml @@ -10,6 +10,9 @@ - name: kubelet-podresources-sock hostPath: path: /var/lib/kubelet/pod-resources/kubelet.sock + - name: nfd-topology-updater-conf + configMap: + name: nfd-topology-updater-conf - op: add path: /spec/template/spec/containers/0/volumeMounts @@ -20,6 +23,9 @@ mountPath: /host-var/lib/kubelet/pod-resources/kubelet.sock - name: host-sys mountPath: /host-sys + - name: nfd-topology-updater-conf + mountPath: "/etc/kubernetes/node-feature-discovery" + readOnly: true - op: add path: /spec/template/spec/containers/0/args/- diff --git a/deployment/overlays/master-worker-topologyupdater/kustomization.yaml b/deployment/overlays/master-worker-topologyupdater/kustomization.yaml index 0725d0c56a..ffcbab6581 100644 --- a/deployment/overlays/master-worker-topologyupdater/kustomization.yaml +++ b/deployment/overlays/master-worker-topologyupdater/kustomization.yaml @@ -19,3 +19,4 @@ components: - ../../components/worker-config - ../../components/common - ../../components/topology-updater +- ../../components/topology-updater-config diff --git a/deployment/overlays/topologyupdater/kustomization.yaml b/deployment/overlays/topologyupdater/kustomization.yaml index 37b01ed764..e007b40a0d 100644 --- a/deployment/overlays/topologyupdater/kustomization.yaml +++ b/deployment/overlays/topologyupdater/kustomization.yaml @@ -16,3 +16,4 @@ resources: components: - ../../components/common - ../../components/topology-updater +- ../../components/topology-updater-config From f832a7e4a8394645203f53b559f40353caa17a4d Mon Sep 17 00:00:00 2001 From: Talor Itzhak Date: Wed, 9 Nov 2022 18:17:40 +0200 Subject: [PATCH 11/87] helm: topology-updater: enable the configuration via helm - Add a helm template with a config example for the exclude-list. - Add mount for the topology-updater.conf file - Update the templates Makefile target Signed-off-by: Talor Itzhak --- Makefile | 8 ++++++++ .../templates/nfd-topologyupdater-conf.yaml | 10 ++++++++++ .../templates/topologyupdater.yaml | 10 ++++++++++ deployment/helm/node-feature-discovery/values.yaml | 10 ++++++++++ 4 files changed, 38 insertions(+) create mode 100644 deployment/helm/node-feature-discovery/templates/nfd-topologyupdater-conf.yaml diff --git a/Makefile b/Makefile index 664ff230b6..3a116678ce 100644 --- a/Makefile +++ b/Makefile @@ -119,12 +119,20 @@ templates: @# Need to prepend each line in the sample config with spaces in order to @# fit correctly in the configmap spec. @sed s'/^/ /' deployment/components/worker-config/nfd-worker.conf.example > nfd-worker.conf.tmp + @sed s'/^/ /' deployment/components/topology-updater-config/nfd-topology-updater.conf.example > nfd-topology-updater.conf.tmp @# The sed magic below replaces the block of text between the lines with start and end markers @start=NFD-WORKER-CONF-START-DO-NOT-REMOVE; \ end=NFD-WORKER-CONF-END-DO-NOT-REMOVE; \ sed -e "/$$start/,/$$end/{ /$$start/{ p; r nfd-worker.conf.tmp" \ -e "}; /$$end/p; d }" -i deployment/helm/node-feature-discovery/values.yaml + @start=NFD-TOPOLOGY-UPDATER-CONF-START-DO-NOT-REMOVE; \ + end=NFD-TOPOLOGY-UPDATER-CONF-END-DO-NOT-REMOVE; \ + sed -e "/$$start/,/$$end/{ /$$start/{ p; r nfd-topology-updater.conf.tmp" \ + -e "}; /$$end/p; d }" -i deployment/helm/node-feature-discovery/values.yaml @rm nfd-worker.conf.tmp + @rm nfd-topology-updater.conf.tmp + + .generator.image.stamp: Dockerfile_generator $(IMAGE_BUILD_CMD) \ diff --git a/deployment/helm/node-feature-discovery/templates/nfd-topologyupdater-conf.yaml b/deployment/helm/node-feature-discovery/templates/nfd-topologyupdater-conf.yaml new file mode 100644 index 0000000000..9867f5089c --- /dev/null +++ b/deployment/helm/node-feature-discovery/templates/nfd-topologyupdater-conf.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-topology-updater-conf + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +data: + nfd-topology-updater.conf: |- + {{- .Values.topologyUpdater.config | toYaml | nindent 4 }} diff --git a/deployment/helm/node-feature-discovery/templates/topologyupdater.yaml b/deployment/helm/node-feature-discovery/templates/topologyupdater.yaml index be48c5fe7d..a53a6033c9 100644 --- a/deployment/helm/node-feature-discovery/templates/topologyupdater.yaml +++ b/deployment/helm/node-feature-discovery/templates/topologyupdater.yaml @@ -68,6 +68,9 @@ spec: mountPath: "/etc/kubernetes/node-feature-discovery/certs" readOnly: true {{- end }} + - name: topology-updater-conf + mountPath: "/etc/kubernetes/node-feature-discovery" + readOnly: true resources: {{- toYaml .Values.topologyUpdater.resources | nindent 12 }} @@ -91,12 +94,19 @@ spec: {{- else }} path: /var/lib/kubelet/pod-resources/kubelet.sock {{- end }} + - name: nfd-topology-updater-conf + configMap: + name: {{ include "node-feature-discovery.fullname" . }}-topology-updater-conf + items: + - key: nfd-topology-updater.conf + path: nfd-topology-updater.conf {{- if .Values.tls.enable }} - name: nfd-topology-updater-cert secret: secretName: nfd-topology-updater-cert {{- end }} + {{- with .Values.topologyUpdater.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/deployment/helm/node-feature-discovery/values.yaml b/deployment/helm/node-feature-discovery/values.yaml index a052a8baa0..477f162975 100644 --- a/deployment/helm/node-feature-discovery/values.yaml +++ b/deployment/helm/node-feature-discovery/values.yaml @@ -368,6 +368,16 @@ worker: priorityClassName: "" topologyUpdater: + config: ### + ## key = node name, value = list of resources to be excluded. + ## use * to exclude from all nodes. + ## an example for how the exclude list should looks like + #excludeList: + # node1: [cpu] + # node2: [memory, example/deviceA] + # *: [hugepages-2Mi] +### + enable: false createCRDs: false From d495376f069cf715e0a0c8e2e5d9606ed1eb9594 Mon Sep 17 00:00:00 2001 From: Talor Itzhak Date: Wed, 9 Nov 2022 19:25:58 +0200 Subject: [PATCH 12/87] docs: topology-updater: update docs for exclude-list feature Update the docs with explanations and examples about the exclude-list feature. Signed-off-by: Talor Itzhak --- docs/deployment/helm.md | 43 +++++++-------- .../topology-updater-commandline-reference.md | 16 +++++- ...opology-updater-configuration-reference.md | 52 +++++++++++++++++++ docs/usage/nfd-topology-updater.md | 39 +++++++++++++- docs/usage/nfd-worker.md | 2 +- 5 files changed, 128 insertions(+), 24 deletions(-) create mode 100644 docs/reference/topology-updater-configuration-reference.md diff --git a/docs/deployment/helm.md b/docs/deployment/helm.md index 0cd5842448..824f3c72bf 100644 --- a/docs/deployment/helm.md +++ b/docs/deployment/helm.md @@ -142,24 +142,25 @@ We have introduced the following Chart parameters. ### Topology updater parameters -| Name | Type | Default | description | -|-----------------------------------------------|--------|---------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `topologyUpdater.*` | dict | | NFD Topology Updater configuration | -| `topologyUpdater.enable` | bool | false | Specifies whether the NFD Topology Updater should be created | -| `topologyUpdater.createCRDs` | bool | false | Specifies whether the NFD Topology Updater CRDs should be created | -| `topologyUpdater.serviceAccount.create` | bool | true | Specifies whether the service account for topology updater should be created | -| `topologyUpdater.serviceAccount.annotations` | dict | {} | Annotations to add to the service account for topology updater | -| `topologyUpdater.serviceAccount.name` | string | | The name of the service account for topology updater to use. If not set and create is true, a name is generated using the fullname template and `-topology-updater` suffix | -| `topologyUpdater.rbac` | dict | | RBAC [parameters](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) for the topology updater | -| `topologyUpdater.rbac.create` | bool | false | Specifies whether the cluster role and binding for topology updater should be created | -| `topologyUpdater.kubeletConfigPath` | string | "" | Specifies the kubelet config host path | -| `topologyUpdater.kubeletPodResourcesSockPath` | string | "" | Specifies the kubelet sock path to read pod resources | -| `topologyUpdater.updateInterval` | string | 60s | Time to sleep between CR updates. Non-positive value implies no CR update. | -| `topologyUpdater.watchNamespace` | string | `*` | Namespace to watch pods, `*` for all namespaces | -| `topologyUpdater.podSecurityContext` | dict | {} | [PodSecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) holds pod-level security attributes and common container settings | -| `topologyUpdater.securityContext` | dict | {} | Container [security settings](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) | -| `topologyUpdater.resources` | dict | {} | Topology updater pod [resources management](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) | -| `topologyUpdater.nodeSelector` | dict | {} | Topology updater pod [node selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) | -| `topologyUpdater.tolerations` | dict | {} | Topology updater pod [node tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) | -| `topologyUpdater.annotations` | dict | {} | Topology updater pod [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) | -| `topologyUpdater.affinity` | dict | {} | Topology updater pod [affinity](https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes-using-node-affinity/) | +| Name | Type | Default | description | +|-----------------------------------------------|--------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `topologyUpdater.*` | dict | | NFD Topology Updater configuration | +| `topologyUpdater.enable` | bool | false | Specifies whether the NFD Topology Updater should be created | +| `topologyUpdater.createCRDs` | bool | false | Specifies whether the NFD Topology Updater CRDs should be created | +| `topologyUpdater.serviceAccount.create` | bool | true | Specifies whether the service account for topology updater should be created | +| `topologyUpdater.serviceAccount.annotations` | dict | {} | Annotations to add to the service account for topology updater | +| `topologyUpdater.serviceAccount.name` | string | | The name of the service account for topology updater to use. If not set and create is true, a name is generated using the fullname template and `-topology-updater` suffix | +| `topologyUpdater.rbac` | dict | | RBAC [parameters](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) for the topology updater | +| `topologyUpdater.rbac.create` | bool | false | Specifies whether the cluster role and binding for topology updater should be created | +| `topologyUpdater.kubeletConfigPath` | string | "" | Specifies the kubelet config host path | +| `topologyUpdater.kubeletPodResourcesSockPath` | string | "" | Specifies the kubelet sock path to read pod resources | +| `topologyUpdater.updateInterval` | string | 60s | Time to sleep between CR updates. Non-positive value implies no CR update. | +| `topologyUpdater.watchNamespace` | string | `*` | Namespace to watch pods, `*` for all namespaces | +| `topologyUpdater.podSecurityContext` | dict | {} | [PodSecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) holds pod-level security attributes and common container settings | +| `topologyUpdater.securityContext` | dict | {} | Container [security settings](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) | +| `topologyUpdater.resources` | dict | {} | Topology updater pod [resources management](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) | +| `topologyUpdater.nodeSelector` | dict | {} | Topology updater pod [node selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) | +| `topologyUpdater.tolerations` | dict | {} | Topology updater pod [node tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) | +| `topologyUpdater.annotations` | dict | {} | Topology updater pod [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) | +| `topologyUpdater.affinity` | dict | {} | Topology updater pod [affinity](https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes-using-node-affinity/) | +| `topologyUpdater.config` | dict | | [configuration](../reference/topology-updater-configuration-reference) | diff --git a/docs/reference/topology-updater-commandline-reference.md b/docs/reference/topology-updater-commandline-reference.md index 742d36c66d..197d950421 100644 --- a/docs/reference/topology-updater-commandline-reference.md +++ b/docs/reference/topology-updater-commandline-reference.md @@ -21,7 +21,8 @@ To quickly view available command line flags execute `nfd-topology-updater -help In a docker container: ```bash -docker run gcr.io/k8s-staging-nfd/node-feature-discovery:master nfd-topology-updater -help +docker run {{ site.container_image }} \ +nfd-topology-updater -help ``` ### -h, -help @@ -32,6 +33,19 @@ Print usage and exit. Print version and exit. +### -config + +The `-config` flag specifies the path of the nfd-topology-updater +configuration file to use. + +Default: /etc/kubernetes/node-feature-discovery/nfd-topology-updater.conf + +Example: + +```bash +nfd-topology-updater -config=/opt/nfd/nfd-topology-updater.conf +``` + ### -server The `-server` flag specifies the address of the nfd-master endpoint where to diff --git a/docs/reference/topology-updater-configuration-reference.md b/docs/reference/topology-updater-configuration-reference.md new file mode 100644 index 0000000000..cda0814520 --- /dev/null +++ b/docs/reference/topology-updater-configuration-reference.md @@ -0,0 +1,52 @@ +--- +title: "Topology-Updater config reference" +layout: default +sort: 5 +--- + +# Configuration file reference of nfd-topology-updater +{: .no_toc} + +## Table of contents +{: .no_toc .text-delta} + +1. TOC +{:toc} + +--- + +See the +[sample configuration file](https://github.com/kubernetes-sigs/node-feature-discovery/blob/{{site.release}}/deployment/components/topology-updater-config/nfd-topology-updater.conf.example) +for a full example configuration. + +## excludeList + +The `excludeList` specifies a key-value map of allocated resources +that should not be examined by the topology-updater +agent per node. +Each key is a node name with a value as a list of resources +that should not be examined by the agent for that specific node. + +Default: *empty* + +Example: + +```yaml +excludeList: + nodeA: [hugepages-2Mi] + nodeB: [memory] + nodeC: [cpu, hugepages-2Mi] +``` + +### excludeList.* +`excludeList.*` is a special value that use to specify all nodes. +A resource that would be listed under this key, would be excluded from all nodes. + +Default: *empty* + +Example: + +```yaml +excludeList: + '*': [hugepages-2Mi] +``` diff --git a/docs/usage/nfd-topology-updater.md b/docs/usage/nfd-topology-updater.md index fab56715c1..353b43d9a9 100644 --- a/docs/usage/nfd-topology-updater.md +++ b/docs/usage/nfd-topology-updater.md @@ -20,5 +20,42 @@ When run as a daemonset, nodes are re-examined for the allocated resources (to determine the information of the allocatable resources on a per zone basis where a zone can be a NUMA node) at an interval specified using the [`-sleep-interval`](../reference/topology-updater-commandline-reference.html#-sleep-interval) -option. The default sleep interval is set to 60s which is the the value when no +option. The default sleep interval is set to 60s which is the value when no -sleep-interval is specified. +In addition, it can avoid examining specific allocated resources +given a configuration of resources to exclude via [`-excludeList`](../reference/topology-updater-configuration-reference.md#excludelist) + +## Topology-Updater Configuration + +NFD-Topology-Updater supports configuration through a configuration file. The +default location is `/etc/kubernetes/node-feature-discovery/topology-updater.conf`, +but, this can be changed by specifying the`-config` command line flag. +> NOTE: unlike nfd-worker, +> dynamic configuration updates are not currently supported. + +Topology-Updater configuration file is read inside the container, +and thus, Volumes and VolumeMounts are needed +to make your configuration available for NFD. +The preferred method is to use a ConfigMap +which provides easy deployment and re-configurability. + +The provided nfd-topology-updater deployment templates +create an empty configmap +and mount it inside the nfd-topology-updater containers. +In kustomize deployments, configuration can be edited with: + +```bash +kubectl -n ${NFD_NS} edit configmap nfd-topology-updater-conf +``` + +In Helm deployments, +[Topology Updater parameters](../deployment/helm.md#topology-updater-parameters) +`toplogyUpdater.config` can be used to edit the respective configuration. + +See +[nfd-topology-updater configuration file reference](../reference/topology-updater-configuration-reference.md) +for more details. +The (empty-by-default) +[example config](https://github.com/kubernetes-sigs/node-feature-discovery/blob/{{site.release}}/deployment/components/topology-updater-config/nfd-topology-updater.conf.example) +contains all available configuration options and can be used as a reference +for creating a configuration. \ No newline at end of file diff --git a/docs/usage/nfd-worker.md b/docs/usage/nfd-worker.md index 6a22eb330d..4a662dbf7c 100644 --- a/docs/usage/nfd-worker.md +++ b/docs/usage/nfd-worker.md @@ -62,7 +62,7 @@ for more details. The (empty-by-default) [example config](https://github.com/kubernetes-sigs/node-feature-discovery/blob/{{site.release}}/deployment/components/worker-config/nfd-worker.conf.example) contains all available configuration options and can be used as a reference -for creating creating a configuration. +for creating a configuration. Configuration options can also be specified via the `-options` command line flag, in which case no mounts need to be used. The same format as in the config From 1719ce6736a29955d3947595b675fe8a510963f6 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Tue, 22 Nov 2022 17:50:54 +0200 Subject: [PATCH 13/87] test/e2e: more flexible pod spec generation Change the pod spec generator functions to accept parameterization in the form of more generic "mutator functions". This makes the addition of new test specific pod spec customizations a lot cleaner. Plus, hopefully makes the code a bit more readable as well. Also, slightly simplify the SpecWithConfigMap() but dropping one redundant argument. Inspired by latest contributions by Talor Itzhak (titzhak@redhat.com). --- test/e2e/node_feature_discovery.go | 66 ++++++------------- test/e2e/topology_updater.go | 14 ++-- test/e2e/utils/pod.go | 102 +++++++++++++++++++---------- 3 files changed, 95 insertions(+), 87 deletions(-) diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index f166ca3e7b..f7bf5b78d3 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -112,8 +112,8 @@ var _ = SIGDescribe("Node Feature Discovery", func() { // Launch nfd-master By("Creating nfd master pod and nfd-master service") - image := fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag) - masterPod = f.PodClient().CreateSync(testutils.NFDMasterPod(image, false)) + imageOpt := testutils.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)) + masterPod = f.PodClient().CreateSync(testutils.NFDMasterPod(imageOpt)) // Create nfd-master service nfdSvc, err := testutils.CreateService(f.ClientSet, f.Namespace.Name) @@ -155,8 +155,11 @@ var _ = SIGDescribe("Node Feature Discovery", func() { // Launch nfd-worker By("Creating a nfd worker pod") - image := fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag) - workerPod := testutils.NFDWorkerPod(image, []string{"-oneshot", "-label-sources=fake"}) + podSpecOpts := []testutils.PodSpecOption{ + testutils.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)), + testutils.SpecWithContainerExtraArgs("-oneshot", "-label-sources=fake"), + } + workerPod := testutils.NFDWorkerPod(podSpecOpts...) workerPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), workerPod, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -202,7 +205,8 @@ var _ = SIGDescribe("Node Feature Discovery", func() { fConf := cfg.DefaultFeatures By("Creating nfd-worker daemonset") - workerDS := testutils.NFDWorkerDaemonSet(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag), []string{}) + podSpecOpts := []testutils.PodSpecOption{testutils.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag))} + workerDS := testutils.NFDWorkerDaemonSet(podSpecOpts...) workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -343,45 +347,12 @@ var _ = SIGDescribe("Node Feature Discovery", func() { Expect(err).NotTo(HaveOccurred()) By("Creating nfd-worker daemonset with configmap mounted") - workerDS := testutils.NFDWorkerDaemonSet(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag), []string{}) - - // add configmap mount config - volumeName1 := "custom-configs-extra1" - volumeName2 := "custom-configs-extra2" - workerDS.Spec.Template.Spec.Volumes = append(workerDS.Spec.Template.Spec.Volumes, - corev1.Volume{ - Name: volumeName1, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: cm1.Name, - }, - }, - }, - }, - corev1.Volume{ - Name: volumeName2, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: cm2.Name, - }, - }, - }, - }, - ) - workerDS.Spec.Template.Spec.Containers[0].VolumeMounts = append(workerDS.Spec.Template.Spec.Containers[0].VolumeMounts, - corev1.VolumeMount{ - Name: volumeName1, - ReadOnly: true, - MountPath: filepath.Join(custom.Directory, "cm1"), - }, - corev1.VolumeMount{ - Name: volumeName2, - ReadOnly: true, - MountPath: filepath.Join(custom.Directory, "cm2"), - }, - ) + podSpecOpts := []testutils.PodSpecOption{ + testutils.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)), + testutils.SpecWithConfigMap(cm1.Name, filepath.Join(custom.Directory, "cm1")), + testutils.SpecWithConfigMap(cm2.Name, filepath.Join(custom.Directory, "cm2")), + } + workerDS := testutils.NFDWorkerDaemonSet(podSpecOpts...) workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -449,8 +420,11 @@ var _ = SIGDescribe("Node Feature Discovery", func() { It("custom labels from the NodeFeatureRule rules should be created", func() { By("Creating nfd-worker daemonset") - workerArgs := []string{"-feature-sources=fake", "-label-sources=", "-sleep-interval=1s"} - workerDS := testutils.NFDWorkerDaemonSet(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag), workerArgs) + podSpecOpts := []testutils.PodSpecOption{ + testutils.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)), + testutils.SpecWithContainerExtraArgs("-feature-sources=fake", "-label-sources=", "-sleep-interval=1s"), + } + workerDS := testutils.NFDWorkerDaemonSet(podSpecOpts...) workerDS, err := f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) diff --git a/test/e2e/topology_updater.go b/test/e2e/topology_updater.go index 375c84fead..4e038d868a 100644 --- a/test/e2e/topology_updater.go +++ b/test/e2e/topology_updater.go @@ -71,8 +71,8 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { Expect(testutils.ConfigureRBAC(f.ClientSet, f.Namespace.Name)).NotTo(HaveOccurred()) - image := fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag) - f.PodClient().CreateSync(testutils.NFDMasterPod(image, false)) + imageOpt := testutils.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)) + f.PodClient().CreateSync(testutils.NFDMasterPod(imageOpt)) // Create nfd-master service masterService, err := testutils.CreateService(f.ClientSet, f.Namespace.Name) @@ -119,7 +119,8 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { kcfg := cfg.GetKubeletConfig() By(fmt.Sprintf("Using config (%#v)", kcfg)) - topologyUpdaterDaemonSet = testutils.NFDTopologyUpdaterDaemonSet(kcfg, fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag), []string{}) + podSpecOpts := []testutils.PodSpecOption{testutils.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag))} + topologyUpdaterDaemonSet = testutils.NFDTopologyUpdaterDaemonSet(kcfg, podSpecOpts...) }) It("should fill the node resource topologies CR with the data", func() { @@ -282,8 +283,11 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { kcfg := cfg.GetKubeletConfig() By(fmt.Sprintf("Using config (%#v)", kcfg)) - opts := testutils.SpecWithConfigMap(cm.Name, cm.Name, "/etc/kubernetes/node-feature-discovery") - topologyUpdaterDaemonSet = testutils.NFDTopologyUpdaterDaemonSet(kcfg, fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag), []string{}, opts) + podSpecOpts := []testutils.PodSpecOption{ + testutils.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)), + testutils.SpecWithConfigMap(cm.Name, "/etc/kubernetes/node-feature-discovery"), + } + topologyUpdaterDaemonSet = testutils.NFDTopologyUpdaterDaemonSet(kcfg, podSpecOpts...) }) It("noderesourcetopology should not advertise the memory resource", func() { diff --git a/test/e2e/utils/pod.go b/test/e2e/utils/pod.go index f1db373b9b..ace9373016 100644 --- a/test/e2e/utils/pod.go +++ b/test/e2e/utils/pod.go @@ -111,8 +111,10 @@ func DeletePodSyncByName(f *framework.Framework, podName string) { f.PodClient().DeleteSync(podName, delOpts, framework.DefaultPodDeletionTimeout) } +type PodSpecOption func(spec *corev1.PodSpec) + // NFDMasterPod provide NFD master pod definition -func NFDMasterPod(image string, onMasterNode bool) *corev1.Pod { +func NFDMasterPod(opts ...PodSpecOption) *corev1.Pod { p := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "nfd-master-", @@ -122,7 +124,6 @@ func NFDMasterPod(image string, onMasterNode bool) *corev1.Pod { Containers: []corev1.Container{ { Name: "node-feature-discovery", - Image: image, ImagePullPolicy: pullPolicy(), Command: []string{"nfd-master"}, Env: []corev1.EnvVar{ @@ -141,27 +142,20 @@ func NFDMasterPod(image string, onMasterNode bool) *corev1.Pod { RestartPolicy: corev1.RestartPolicyNever, }, } - if onMasterNode { - p.Spec.NodeSelector = map[string]string{"node-role.kubernetes.io/master": ""} - p.Spec.Tolerations = []corev1.Toleration{ - { - Key: "node-role.kubernetes.io/master", - Operator: corev1.TolerationOpEqual, - Value: "", - Effect: corev1.TaintEffectNoSchedule, - }, - } + + for _, o := range opts { + o(&p.Spec) } return p } // NFDWorkerPod provides NFD worker pod definition -func NFDWorkerPod(image string, extraArgs []string) *corev1.Pod { +func NFDWorkerPod(opts ...PodSpecOption) *corev1.Pod { p := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "nfd-worker-" + string(uuid.NewUUID()), }, - Spec: *nfdWorkerPodSpec(image, extraArgs), + Spec: *nfdWorkerPodSpec(opts...), } p.Spec.RestartPolicy = corev1.RestartPolicyNever @@ -170,29 +164,58 @@ func NFDWorkerPod(image string, extraArgs []string) *corev1.Pod { } // NFDWorkerDaemonSet provides the NFD daemon set worker definition -func NFDWorkerDaemonSet(image string, extraArgs []string) *appsv1.DaemonSet { - podSpec := nfdWorkerPodSpec(image, extraArgs) - return newDaemonSet("nfd-worker", podSpec) +func NFDWorkerDaemonSet(opts ...PodSpecOption) *appsv1.DaemonSet { + return newDaemonSet("nfd-worker", nfdWorkerPodSpec(opts...)) } // NFDTopologyUpdaterDaemonSet provides the NFD daemon set topology updater -func NFDTopologyUpdaterDaemonSet(kc KubeletConfig, image string, extraArgs []string, options ...func(spec *corev1.PodSpec)) *appsv1.DaemonSet { - podSpec := nfdTopologyUpdaterPodSpec(kc, image, extraArgs) - for _, o := range options { - o(podSpec) +func NFDTopologyUpdaterDaemonSet(kc KubeletConfig, opts ...PodSpecOption) *appsv1.DaemonSet { + return newDaemonSet("nfd-topology-updater", nfdTopologyUpdaterPodSpec(kc, opts...)) +} + +// SpecWithContainerImage returns a PodSpecOption that sets the image used by the first container. +func SpecWithContainerImage(image string) PodSpecOption { + return func(spec *corev1.PodSpec) { + // NOTE: we might want to make the container number a parameter + cnt := &spec.Containers[0] + cnt.Image = image } - return newDaemonSet("nfd-topology-updater", podSpec) } -func SpecWithConfigMap(cmName, volumeName, mountPath string) func(spec *corev1.PodSpec) { +// SpecWithContainerExtraArgs returns a PodSpecOption that adds extra args to the first container. +func SpecWithContainerExtraArgs(args ...string) PodSpecOption { + return func(spec *corev1.PodSpec) { + // NOTE: we might want to make the container number a parameter + cnt := &spec.Containers[0] + cnt.Args = append(cnt.Args, args...) + } +} + +// SpecWithMasterNodeSelector returns a PodSpecOption that modifies the pod to +// be run on a control plane node of the cluster. +func SpecWithMasterNodeSelector(args ...string) PodSpecOption { + return func(spec *corev1.PodSpec) { + spec.NodeSelector["node-role.kubernetes.io/control-plane"] = "" + spec.Tolerations = append(spec.Tolerations, + corev1.Toleration{ + Key: "node-role.kubernetes.io/control-plane", + Operator: corev1.TolerationOpEqual, + Value: "", + Effect: corev1.TaintEffectNoSchedule, + }) + } +} + +// SpecWithConfigMap returns a PodSpecOption that mounts a configmap to the first container. +func SpecWithConfigMap(name, mountPath string) PodSpecOption { return func(spec *corev1.PodSpec) { spec.Volumes = append(spec.Volumes, corev1.Volume{ - Name: volumeName, + Name: name, VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: cmName, + Name: name, }, }, }, @@ -200,7 +223,7 @@ func SpecWithConfigMap(cmName, volumeName, mountPath string) func(spec *corev1.P cnt := &spec.Containers[0] cnt.VolumeMounts = append(cnt.VolumeMounts, corev1.VolumeMount{ - Name: volumeName, + Name: name, ReadOnly: true, MountPath: mountPath, }) @@ -228,17 +251,16 @@ func newDaemonSet(name string, podSpec *corev1.PodSpec) *appsv1.DaemonSet { } } -func nfdWorkerPodSpec(image string, extraArgs []string) *corev1.PodSpec { +func nfdWorkerPodSpec(opts ...PodSpecOption) *corev1.PodSpec { yes := true no := false - return &corev1.PodSpec{ + p := &corev1.PodSpec{ Containers: []corev1.Container{ { Name: "node-feature-discovery", - Image: image, ImagePullPolicy: pullPolicy(), Command: []string{"nfd-worker"}, - Args: append([]string{"-server=nfd-master-e2e:8080"}, extraArgs...), + Args: []string{"-server=nfd-master-e2e:8080"}, Env: []corev1.EnvVar{ { Name: "NODE_NAME", @@ -337,23 +359,26 @@ func nfdWorkerPodSpec(image string, extraArgs []string) *corev1.PodSpec { }, }, } + + for _, o := range opts { + o(p) + } + return p } -func nfdTopologyUpdaterPodSpec(kc KubeletConfig, image string, extraArgs []string) *corev1.PodSpec { - return &corev1.PodSpec{ +func nfdTopologyUpdaterPodSpec(kc KubeletConfig, opts ...PodSpecOption) *corev1.PodSpec { + p := &corev1.PodSpec{ Containers: []corev1.Container{ { Name: "node-topology-updater", - Image: image, ImagePullPolicy: pullPolicy(), Command: []string{"nfd-topology-updater"}, - Args: append([]string{ + Args: []string{ "--kubelet-config-uri=file:///podresources/config.yaml", "--podresources-socket=unix:///podresources/kubelet.sock", "--sleep-interval=3s", "--watch-namespace=rte", - "--server=nfd-master-e2e:8080", - }, extraArgs...), + "--server=nfd-master-e2e:8080"}, Env: []corev1.EnvVar{ { Name: "NODE_NAME", @@ -420,6 +445,11 @@ func nfdTopologyUpdaterPodSpec(kc KubeletConfig, image string, extraArgs []strin }, }, } + + for _, o := range opts { + o(p) + } + return p } func newHostPathType(typ corev1.HostPathType) *corev1.HostPathType { From eb8e29c80a53ce6eaa844f5316b432d90074cb16 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Tue, 22 Nov 2022 12:36:32 +0200 Subject: [PATCH 14/87] nfd-worker: drop deprecated command line flags Drop the following flags that were deprecated already in v0.8.0: -sleep-interval (replaced by core.sleepInterval config file option) -label-whitelist (replaced by core.labelWhiteList config file option) -sources (replaced by -label-sources flag) --- cmd/nfd-worker/main.go | 21 --------- cmd/nfd-worker/main_test.go | 9 +--- .../reference/worker-commandline-reference.md | 47 ------------------- .../worker-configuration-reference.md | 10 +--- docs/usage/customization-guide.md | 4 +- docs/usage/features.md | 8 +++- docs/usage/nfd-worker.md | 4 +- .../worker/nfd-worker-internal_test.go | 22 ++------- pkg/nfd-client/worker/nfd-worker.go | 9 ---- 9 files changed, 14 insertions(+), 120 deletions(-) diff --git a/cmd/nfd-worker/main.go b/cmd/nfd-worker/main.go index c8afc0abde..90892380f1 100644 --- a/cmd/nfd-worker/main.go +++ b/cmd/nfd-worker/main.go @@ -84,15 +84,6 @@ func parseArgs(flags *flag.FlagSet, osArgs ...string) *worker.Args { args.Overrides.FeatureSources = overrides.FeatureSources case "label-sources": args.Overrides.LabelSources = overrides.LabelSources - case "label-whitelist": - klog.Warningf("-label-whitelist is deprecated, use 'core.labelWhiteList' option in the config file, instead") - args.Overrides.LabelWhiteList = overrides.LabelWhiteList - case "sleep-interval": - klog.Warningf("-sleep-interval is deprecated, use 'core.sleepInterval' option in the config file, instead") - args.Overrides.SleepInterval = overrides.SleepInterval - case "sources": - klog.Warningf("-sources is deprecated, use '-label-sources' flag, instead") - args.Overrides.LabelSources = overrides.LabelSources } }) @@ -124,7 +115,6 @@ func initFlags(flagset *flag.FlagSet) (*worker.Args, *worker.ConfigOverrideArgs) // Flags overlapping with config file options overrides := &worker.ConfigOverrideArgs{ - LabelWhiteList: &utils.RegexpVal{}, FeatureSources: &utils.StringSliceVal{}, LabelSources: &utils.StringSliceVal{}, } @@ -136,17 +126,6 @@ func initFlags(flagset *flag.FlagSet) (*worker.Args, *worker.ConfigOverrideArgs) flagset.Var(overrides.LabelSources, "label-sources", "Comma separated list of label sources. Special value 'all' enables all sources. "+ "Prefix the source name with '-' to disable it.") - flagset.Var(overrides.LabelWhiteList, "label-whitelist", - "Regular expression to filter label names to publish to the Kubernetes API server. "+ - "NB: the label namespace is omitted i.e. the filter is only applied to the name part after '/'. "+ - "DEPRECATED: This parameter should be set via the config file.") - overrides.SleepInterval = flagset.Duration("sleep-interval", 0, - "Time to sleep between re-labeling. Non-positive value implies no re-labeling (i.e. infinite sleep). "+ - "DEPRECATED: This parameter should be set via the config file") - flagset.Var(overrides.LabelSources, "sources", - "Comma separated list of label sources. Special value 'all' enables all feature sources. "+ - "Prefix the source name with '-' to disable it. "+ - "DEPRECATED: use -label-sources instead") return args, overrides } diff --git a/cmd/nfd-worker/main_test.go b/cmd/nfd-worker/main_test.go index 4f3398ed10..68c425c504 100644 --- a/cmd/nfd-worker/main_test.go +++ b/cmd/nfd-worker/main_test.go @@ -19,7 +19,6 @@ package main import ( "flag" "testing" - "time" . "github.com/smartystreets/goconvey/convey" @@ -36,8 +35,6 @@ func TestParseArgs(t *testing.T) { Convey("overrides should be nil", func() { So(args.Oneshot, ShouldBeTrue) So(args.Overrides.NoPublish, ShouldBeNil) - So(args.Overrides.LabelWhiteList, ShouldBeNil) - So(args.Overrides.SleepInterval, ShouldBeNil) So(args.Overrides.FeatureSources, ShouldBeNil) So(args.Overrides.LabelSources, ShouldBeNil) }) @@ -46,18 +43,14 @@ func TestParseArgs(t *testing.T) { Convey("When all override args are specified", func() { args := parseArgs(flags, "-no-publish", - "-label-whitelist=.*rdt.*", "-feature-sources=cpu", - "-label-sources=fake1,fake2,fake3", - "-sleep-interval=30s") + "-label-sources=fake1,fake2,fake3") Convey("args.sources is set to appropriate values", func() { So(args.Oneshot, ShouldBeFalse) So(*args.Overrides.NoPublish, ShouldBeTrue) - So(*args.Overrides.SleepInterval, ShouldEqual, 30*time.Second) So(*args.Overrides.FeatureSources, ShouldResemble, utils.StringSliceVal{"cpu"}) So(*args.Overrides.LabelSources, ShouldResemble, utils.StringSliceVal{"fake1", "fake2", "fake3"}) - So(args.Overrides.LabelWhiteList.Regexp.String(), ShouldResemble, ".*rdt.*") }) }) }) diff --git a/docs/reference/worker-commandline-reference.md b/docs/reference/worker-commandline-reference.md index 9b3383447a..886faf4c7e 100644 --- a/docs/reference/worker-commandline-reference.md +++ b/docs/reference/worker-commandline-reference.md @@ -178,10 +178,6 @@ Example: nfd-worker -label-sources=kernel,system,local ``` -### -sources - -**DEPRECATED**: use [`-label-sources`](#-label-sources) instead. - ### -no-publish The `-no-publish` flag disables all communication with the nfd-master, making @@ -196,29 +192,6 @@ Example: nfd-worker -no-publish ``` -### -label-whitelist - -The `-label-whitelist` specifies a regular expression for filtering feature -labels based on their name. Each label must match against the given reqular -expression in order to be published. - -Note: The regular expression is only matches against the "basename" part of the -label, i.e. to the part of the name after '/'. The label namespace is omitted. - -Note: This flag takes precedence over the `core.labelWhiteList` configuration -file option. - -Default: *empty* - -Example: - -```bash -nfd-worker -label-whitelist='.*cpuid\.' -``` - -**DEPRECATED**: you should use the `core.labelWhiteList` option in the -configuration file, instead. - ### -oneshot The `-oneshot` flag causes nfd-worker to exit after one pass of feature @@ -232,26 +205,6 @@ Example: nfd-worker -oneshot -no-publish ``` -### -sleep-interval - -The `-sleep-interval` specifies the interval between feature re-detection (and -node re-labeling). A non-positive value implies infinite sleep interval, i.e. -no re-detection or re-labeling is done. - -Note: This flag takes precedence over the `core.sleepInterval` configuration -file option. - -Default: 60s - -Example: - -```bash -nfd-worker -sleep-interval=1h -``` - -**DEPRECATED**: you should use the `core.sleepInterval` option in the -configuration file, instead. - ### Logging The following logging-related flags are inherited from the diff --git a/docs/reference/worker-configuration-reference.md b/docs/reference/worker-configuration-reference.md index bcdf206021..5dd239e811 100644 --- a/docs/reference/worker-configuration-reference.md +++ b/docs/reference/worker-configuration-reference.md @@ -31,9 +31,6 @@ feature (re-)detection, and thus also the interval between node re-labeling. A non-positive value implies infinite sleep interval, i.e. no re-detection or re-labeling is done. -Note: Overridden by the deprecated `-sleep-interval` command line flag (if -specified). - Default: `60s` Example: @@ -82,8 +79,8 @@ conjunction with `all`. This configuration option affects the generation of node labels but not the actual discovery of the underlying feature data that is used e.g. in custom/`NodeFeatureRule` rules. -Note: Overridden by the `-label-sources` and `-sources` command line flags and -the `core.sources` configurations option (if any of them is specified). +Note: Overridden by the `-label-sources` command line flag and +the `core.sources` configurations option (if either of them is specified). Default: `[all]` @@ -122,9 +119,6 @@ Note: The regular expression is only matches against the "basename" part of the label, i.e. to the part of the name after '/'. The label prefix (or namespace) is omitted. -Note: Overridden by the deprecated `-label-whitelist` command line flag (if -specified). - Default: `null` Example: diff --git a/docs/usage/customization-guide.md b/docs/usage/customization-guide.md index cb2d48ff9c..640cf7ac24 100644 --- a/docs/usage/customization-guide.md +++ b/docs/usage/customization-guide.md @@ -84,9 +84,7 @@ labels, accordingly. is received from nfd-worker. That is, in practice rules are evaluated and labels for each node are created on intervals specified by the [`core.sleepInterval`](../reference/worker-configuration-reference#coresleepinterval) -configuration option (or -[`-sleep-interval`](../reference/worker-commandline-reference#-sleep-interval) -command line flag) of nfd-worker instances. This means that modification or +configuration option of nfd-worker instances. This means that modification or creation of `NodeFeatureRule` objects does not instantly cause the node labels to be updated. Instead, the changes only come visible in node labels as nfd-worker instances send their labelling requests. diff --git a/docs/usage/features.md b/docs/usage/features.md index b7ed5ff8ba..24797e8aab 100644 --- a/docs/usage/features.md +++ b/docs/usage/features.md @@ -33,10 +33,14 @@ have the following format. feature.node.kubernetes.io/ = ``` -*Note: Consecutive runs of nfd-worker will update the labels on a +> NOTE: Consecutive runs of nfd-worker will update the labels on a given node. If features are not discovered on a consecutive run, the corresponding label will be removed. This includes any restrictions placed on the consecutive run, -such as restricting discovered features with the -label-whitelist option.* +such as restricting discovered features with the +[`-label-whitelist`](../reference/master-commandline-reference#-label-whitelist) +flag of nfd-master or +[`core.labelWhiteList`](../reference/worker-configuration-reference#corelabelwhitelist) +option of nfd-worker. ### CPU diff --git a/docs/usage/nfd-worker.md b/docs/usage/nfd-worker.md index 4a662dbf7c..de401b2f19 100644 --- a/docs/usage/nfd-worker.md +++ b/docs/usage/nfd-worker.md @@ -23,9 +23,7 @@ Worker connects to the nfd-master service to advertise hardware features. When run as a daemonset, nodes are re-labeled at an default interval of 60s. This can be changed by using the [`core.sleepInterval`](../reference/worker-configuration-reference.html#coresleepinterval) -config option (or -[`-sleep-interval`](../reference/worker-commandline-reference.html#-sleep-interval) -command line flag). +config option. The worker configuration file is watched and re-read on every change which provides a simple mechanism of dynamic run-time reconfiguration. See diff --git a/pkg/nfd-client/worker/nfd-worker-internal_test.go b/pkg/nfd-client/worker/nfd-worker-internal_test.go index a748fdcf4c..7141977123 100644 --- a/pkg/nfd-client/worker/nfd-worker-internal_test.go +++ b/pkg/nfd-client/worker/nfd-worker-internal_test.go @@ -170,18 +170,16 @@ sources: }) Convey("and a proper config file and overrides are given", func() { - sleepIntervalArg := 15 * time.Second - worker.args = Args{Overrides: ConfigOverrideArgs{SleepInterval: &sleepIntervalArg}} + worker.args = Args{Overrides: ConfigOverrideArgs{FeatureSources: &utils.StringSliceVal{"cpu"}}} overrides := `{"core": {"labelSources": ["fake"],"noPublish": true},"sources": {"pci": {"deviceClassWhitelist": ["03"]}}}` So(worker.configure(f.Name(), overrides), ShouldBeNil) Convey("overrides should take precedence over the config file", func() { // Verify core config So(worker.config.Core.NoPublish, ShouldBeTrue) - So(worker.config.Core.FeatureSources, ShouldResemble, []string{"memory", "storage"}) - So(worker.config.Core.LabelSources, ShouldResemble, []string{"fake"}) // from overrides + So(worker.config.Core.FeatureSources, ShouldResemble, []string{"cpu"}) // from cmdline + So(worker.config.Core.LabelSources, ShouldResemble, []string{"fake"}) // from overrides So(worker.config.Core.LabelWhiteList.String(), ShouldEqual, "foo") - So(worker.config.Core.SleepInterval.Duration, ShouldEqual, 15*time.Second) // from cmdline // Verify feature source config So(err, ShouldBeNil) @@ -339,20 +337,6 @@ func TestNewNfdWorker(t *testing.T) { So(worker.config.Core.LabelWhiteList, ShouldResemble, emptyRegexp) }) }) - - Convey("with valid LabelWhiteListStr arg specified", func() { - args := &Args{Overrides: ConfigOverrideArgs{LabelWhiteList: &utils.RegexpVal{Regexp: *regexp.MustCompile(".*rdt.*")}}} - w, err := NewNfdWorker(args) - Convey("no error should be returned", func() { - So(err, ShouldBeNil) - }) - worker := w.(*nfdWorker) - So(worker.configure("", ""), ShouldBeNil) - expectRegexp := utils.RegexpVal{Regexp: *regexp.MustCompile(".*rdt.*")} - Convey("proper labelWhiteList regexp should be produced", func() { - So(worker.config.Core.LabelWhiteList, ShouldResemble, expectRegexp) - }) - }) }) } diff --git a/pkg/nfd-client/worker/nfd-worker.go b/pkg/nfd-client/worker/nfd-worker.go index ba3a5aad2c..e7a6df73af 100644 --- a/pkg/nfd-client/worker/nfd-worker.go +++ b/pkg/nfd-client/worker/nfd-worker.go @@ -88,9 +88,6 @@ type Args struct { type ConfigOverrideArgs struct { NoPublish *bool - // Deprecated - LabelWhiteList *utils.RegexpVal - SleepInterval *time.Duration FeatureSources *utils.StringSliceVal LabelSources *utils.StringSliceVal } @@ -415,15 +412,9 @@ func (w *nfdWorker) configure(filepath string, overrides string) error { return fmt.Errorf("failed to parse -options: %s", err) } - if w.args.Overrides.LabelWhiteList != nil { - c.Core.LabelWhiteList = *w.args.Overrides.LabelWhiteList - } if w.args.Overrides.NoPublish != nil { c.Core.NoPublish = *w.args.Overrides.NoPublish } - if w.args.Overrides.SleepInterval != nil { - c.Core.SleepInterval = duration{*w.args.Overrides.SleepInterval} - } if w.args.Overrides.FeatureSources != nil { c.Core.FeatureSources = *w.args.Overrides.FeatureSources } From 0174a6405b4efc9764555ea95a465335836d776e Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Tue, 22 Nov 2022 20:58:53 +0200 Subject: [PATCH 15/87] test/e2e: add helper for creating new configmaps Implement a simple helper for creating a configmap with one key. One key (one config file) is enough for our current tests. --- test/e2e/node_feature_discovery.go | 21 ++++---------------- test/e2e/topology_updater.go | 17 ++++------------ test/e2e/utils/configmap.go | 32 ++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 30 deletions(-) create mode 100644 test/e2e/utils/configmap.go diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index f7bf5b78d3..e04e946308 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -32,7 +32,6 @@ import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" extclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/uuid" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" e2elog "k8s.io/kubernetes/test/e2e/framework/log" @@ -308,25 +307,18 @@ var _ = SIGDescribe("Node Feature Discovery", func() { targetLabelNameNegative := "nodename-test-negative" // create 2 configmaps - data1 := make(map[string]string) - data1["custom1.conf"] = ` + data1 := ` - name: ` + targetLabelName + ` matchOn: # default value is true - nodename: - ` + targetNodeName - cm1 := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "custom-config-extra-" + string(uuid.NewUUID()), - }, - Data: data1, - } + cm1 := testutils.NewConfigMap("custom-config-extra-1", "custom.conf", data1) cm1, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(context.TODO(), cm1, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - data2 := make(map[string]string) - data2["custom1.conf"] = ` + data2 := ` - name: ` + targetLabelNameWildcard + ` value: ` + targetLabelValueWildcard + ` matchOn: @@ -337,12 +329,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { - nodename: - "thisNameShouldNeverMatch"` - cm2 := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "custom-config-extra-" + string(uuid.NewUUID()), - }, - Data: data2, - } + cm2 := testutils.NewConfigMap("custom-config-extra-2", "custom.conf", data2) cm2, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(context.TODO(), cm2, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) diff --git a/test/e2e/topology_updater.go b/test/e2e/topology_updater.go index 4e038d868a..f8b1954f3e 100644 --- a/test/e2e/topology_updater.go +++ b/test/e2e/topology_updater.go @@ -260,21 +260,12 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { }) When("topology-updater configure to exclude memory", func() { - var topologyUpdaterConfigMap *corev1.ConfigMap - BeforeEach(func() { - data := make(map[string]string) - data["nfd-topology-updater.conf"] = `excludeList: + cm := testutils.NewConfigMap("nfd-topology-updater-conf", "nfd-topology-updater.conf", ` +excludeList: '*': [memory] -` - topologyUpdaterConfigMap = &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "nfd-topology-updater-conf", - }, - Data: data, - } - - cm, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(context.TODO(), topologyUpdaterConfigMap, metav1.CreateOptions{}) +`) + cm, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(context.TODO(), cm, metav1.CreateOptions{}) Expect(err).ToNot(HaveOccurred()) cfg, err := testutils.GetConfig() diff --git a/test/e2e/utils/configmap.go b/test/e2e/utils/configmap.go new file mode 100644 index 0000000000..192d11023e --- /dev/null +++ b/test/e2e/utils/configmap.go @@ -0,0 +1,32 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// CreateConfigMap is a helper for creating a simple ConfigMap object with one key. +func NewConfigMap(name, key, data string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Data: map[string]string{key: data}, + } +} From f31b70dce48ed214a1231ca1d95833763e4cf050 Mon Sep 17 00:00:00 2001 From: Talor Itzhak Date: Thu, 24 Nov 2022 12:37:06 +0200 Subject: [PATCH 16/87] e2e: add SecurityContext to master The master pod need these `SecurityContext` configurations In order to run inside a namespace with restricted policy Signed-off-by: Talor Itzhak --- test/e2e/utils/pod.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/e2e/utils/pod.go b/test/e2e/utils/pod.go index ace9373016..6741f56b64 100644 --- a/test/e2e/utils/pod.go +++ b/test/e2e/utils/pod.go @@ -115,6 +115,8 @@ type PodSpecOption func(spec *corev1.PodSpec) // NFDMasterPod provide NFD master pod definition func NFDMasterPod(opts ...PodSpecOption) *corev1.Pod { + yes := true + no := false p := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "nfd-master-", @@ -136,6 +138,18 @@ func NFDMasterPod(opts ...PodSpecOption) *corev1.Pod { }, }, }, + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + Privileged: &no, + RunAsNonRoot: &yes, + ReadOnlyRootFilesystem: &yes, + AllowPrivilegeEscalation: &no, + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + }, }, }, ServiceAccountName: "nfd-master-e2e", From c1fa8b2f28db67daa11806dc2f7400ff4a1f86cb Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Mon, 28 Nov 2022 17:49:19 +0200 Subject: [PATCH 17/87] docs: revise topology-updater helm chart rbac parameters --- docs/deployment/helm.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/deployment/helm.md b/docs/deployment/helm.md index 824f3c72bf..ff33fe9e55 100644 --- a/docs/deployment/helm.md +++ b/docs/deployment/helm.md @@ -114,7 +114,7 @@ We have introduced the following Chart parameters. | `master.serviceAccount.create` | bool | true | Specifies whether a service account should be created | `master.serviceAccount.annotations` | dict | {} | Annotations to add to the service account | `master.serviceAccount.name` | string | | The name of the service account to use. If not set and create is true, a name is generated using the fullname template -| `master.rbac.create` | bool | true | Specifies whether to create [RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) configuration for nfd-master +| `master.rbac.create` | bool | true | Specifies whether to create [RBAC][rbac] configuration for nfd-master | `master.service.type` | string | ClusterIP | NFD master service type | | `master.service.port` | integer | 8080 | NFD master service port | | `master.resources` | dict | {} | NFD master pod [resources management](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) | @@ -150,8 +150,7 @@ We have introduced the following Chart parameters. | `topologyUpdater.serviceAccount.create` | bool | true | Specifies whether the service account for topology updater should be created | | `topologyUpdater.serviceAccount.annotations` | dict | {} | Annotations to add to the service account for topology updater | | `topologyUpdater.serviceAccount.name` | string | | The name of the service account for topology updater to use. If not set and create is true, a name is generated using the fullname template and `-topology-updater` suffix | -| `topologyUpdater.rbac` | dict | | RBAC [parameters](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) for the topology updater | -| `topologyUpdater.rbac.create` | bool | false | Specifies whether the cluster role and binding for topology updater should be created | +| `topologyUpdater.rbac.create` | bool | false | Specifies whether to create [RBAC][rbac] configuration for topology updater | | `topologyUpdater.kubeletConfigPath` | string | "" | Specifies the kubelet config host path | | `topologyUpdater.kubeletPodResourcesSockPath` | string | "" | Specifies the kubelet sock path to read pod resources | | `topologyUpdater.updateInterval` | string | 60s | Time to sleep between CR updates. Non-positive value implies no CR update. | @@ -164,3 +163,6 @@ We have introduced the following Chart parameters. | `topologyUpdater.annotations` | dict | {} | Topology updater pod [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) | | `topologyUpdater.affinity` | dict | {} | Topology updater pod [affinity](https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes-using-node-affinity/) | | `topologyUpdater.config` | dict | | [configuration](../reference/topology-updater-configuration-reference) | + + +[rbac]: https://kubernetes.io/docs/reference/access-authn-authz/rbac/ From d0a4cf7564fca8f85d2fc4ed80189221da56bba0 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Mon, 28 Nov 2022 18:07:17 +0200 Subject: [PATCH 18/87] docs: document helm chart params related to worker serviceaccount --- docs/deployment/helm.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/deployment/helm.md b/docs/deployment/helm.md index 824f3c72bf..41a3d64bb7 100644 --- a/docs/deployment/helm.md +++ b/docs/deployment/helm.md @@ -132,6 +132,9 @@ We have introduced the following Chart parameters. | `worker.config` | dict | | NFD worker [configuration](../reference/worker-configuration-reference) | | `worker.podSecurityContext` | dict | {} | [PodSecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) holds pod-level security attributes and common container settings | | `worker.securityContext` | dict | {} | Container [security settings](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) | +| `worker.serviceAccount.create` | bool | true | Specifies whether a service account for nfd-worker should be created +| `worker.serviceAccount.annotations` | dict | {} | Annotations to add to the service account for nfd-worker +| `worker.serviceAccount.name` | string | | The name of the service account to use for nfd-worker. If not set and create is true, a name is generated using the fullname template (suffixed with `-worker`) | `worker.mountUsrSrc` | bool | false | Specifies whether to allow users to mount the hostpath /user/src. Does not work on systems without /usr/src AND a read-only /usr | | `worker.resources` | dict | {} | NFD worker pod [resources management](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) | | `worker.nodeSelector` | dict | {} | NFD worker pod [node selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) | From 6f02ef92de30bacf96dbf0835dc644ea203ab4f1 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Tue, 29 Nov 2022 10:19:11 +0200 Subject: [PATCH 19/87] test/e2e: remove dropped -sleep-interval arg Replace with a ConfigMap-based configuration. --- test/e2e/node_feature_discovery.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index e04e946308..93fcefdd2c 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -406,13 +406,23 @@ var _ = SIGDescribe("Node Feature Discovery", func() { }) It("custom labels from the NodeFeatureRule rules should be created", func() { + By("Creating nfd-worker config") + cm := testutils.NewConfigMap("nfd-worker-conf", "nfd-worker.conf", ` +core: + sleepInterval: "1s" + featureSources: ["fake"] + labelSources: [] +`) + cm, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(context.TODO(), cm, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + By("Creating nfd-worker daemonset") podSpecOpts := []testutils.PodSpecOption{ testutils.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)), - testutils.SpecWithContainerExtraArgs("-feature-sources=fake", "-label-sources=", "-sleep-interval=1s"), + testutils.SpecWithConfigMap(cm.Name, "/etc/kubernetes/node-feature-discovery"), } workerDS := testutils.NFDWorkerDaemonSet(podSpecOpts...) - workerDS, err := f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) + workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) By("Waiting for daemonset pods to be ready") From 37d51c96f195fce83d0ba0663e2a7f56767bda10 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Tue, 29 Nov 2022 13:43:11 +0200 Subject: [PATCH 20/87] deployment: drop stale nfd-api-crds.yaml Remove a stale unused file that was accidentally committed from an experimental work. --- deployment/base/nfd-crds/nfd-api-crds.yaml | 334 ------------------ .../crds/nfd-api-crds.yaml | 334 ------------------ 2 files changed, 668 deletions(-) delete mode 100644 deployment/base/nfd-crds/nfd-api-crds.yaml delete mode 100644 deployment/helm/node-feature-discovery/crds/nfd-api-crds.yaml diff --git a/deployment/base/nfd-crds/nfd-api-crds.yaml b/deployment/base/nfd-crds/nfd-api-crds.yaml deleted file mode 100644 index c59796b836..0000000000 --- a/deployment/base/nfd-crds/nfd-api-crds.yaml +++ /dev/null @@ -1,334 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null - name: nodefeatures.nfd.k8s-sigs.io -spec: - group: nfd.k8s-sigs.io - names: - kind: NodeFeature - listKind: NodeFeatureList - plural: nodefeatures - singular: nodefeature - scope: Cluster - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: NodeFeature resource holds the features discovered for one node - in the cluster. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: NodeFeatureSpec describes a NodeFeature object. - properties: - domains: - description: Features is a list per-domain feature sets. This is the - full "raw" features data that nfd-worker has discovered. - items: - description: DomainFeatures is the collection of all discovered - features of one domain. - properties: - attributes: - additionalProperties: - description: AttributeFeatureSet is a set of features having - string value. - properties: - elements: - additionalProperties: - type: string - type: object - required: - - elements - type: object - type: object - flags: - additionalProperties: - description: FlagFeatureSet is a set of simple features only - containing names without values. - properties: - elements: - additionalProperties: - description: Nil is a dummy empty struct for protobuf - compatibility - type: object - type: object - required: - - elements - type: object - type: object - instances: - additionalProperties: - description: InstanceFeatureSet is a set of features each - of which is an instance having multiple attributes. - properties: - elements: - items: - description: InstanceFeature represents one instance - of a complex features, e.g. a device. - properties: - attributes: - additionalProperties: - type: string - type: object - required: - - attributes - type: object - type: array - required: - - elements - type: object - type: object - name: - type: string - required: - - attributes - - flags - - instances - - name - type: object - type: array - labelsRequest: - additionalProperties: - type: string - description: LabelsRequest is the set of node labels that are requested - to be generatd. - type: object - required: - - domains - type: object - required: - - spec - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null - name: nodefeaturerules.nfd.k8s-sigs.io -spec: - group: nfd.k8s-sigs.io - names: - kind: NodeFeatureRule - listKind: NodeFeatureRuleList - plural: nodefeaturerules - singular: nodefeaturerule - scope: Cluster - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: NodeFeatureRule resource specifies a configuration for feature-based - customization of node objects, such as node labeling. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: NodeFeatureRuleSpec describes a NodeFeatureRule. - properties: - rules: - description: Rules is a list of node customization rules. - items: - description: Rule defines a rule for node customization such as - labeling. - properties: - labels: - additionalProperties: - type: string - description: Labels to create if the rule matches. - type: object - labelsTemplate: - description: LabelsTemplate specifies a template to expand for - dynamically generating multiple labels. Data (after template - expansion) must be keys with an optional value ([=]) - separated by newlines. - type: string - matchAny: - description: MatchAny specifies a list of matchers one of which - must match. - items: - description: MatchAnyElem specifies one sub-matcher of MatchAny. - properties: - matchFeatures: - description: MatchFeatures specifies a set of matcher - terms all of which must match. - items: - description: FeatureMatcherTerm defines requirements - against one feature set. All requirements (specified - as MatchExpressions) are evaluated against each element - in the feature set. - properties: - feature: - type: string - matchExpressions: - additionalProperties: - description: "MatchExpression specifies an expression - to evaluate against a set of input values. It - contains an operator that is applied when matching - the input and an array of values that the operator - evaluates the input against. \n NB: CreateMatchExpression - or MustCreateMatchExpression() should be used - for creating new instances. NB: Validate() must - be called if Op or Value fields are modified - or if a new instance is created from scratch - without using the helper functions." - properties: - op: - description: Op is the operator to be applied. - enum: - - In - - NotIn - - InRegexp - - Exists - - DoesNotExist - - Gt - - Lt - - GtLt - - IsTrue - - IsFalse - type: string - value: - description: Value is the list of values that - the operand evaluates the input against. - Value should be empty if the operator is - Exists, DoesNotExist, IsTrue or IsFalse. - Value should contain exactly one element - if the operator is Gt or Lt and exactly - two elements if the operator is GtLt. In - other cases Value should contain at least - one element. - items: - type: string - type: array - required: - - op - type: object - description: MatchExpressionSet contains a set of - MatchExpressions, each of which is evaluated against - a set of input values. - type: object - required: - - feature - - matchExpressions - type: object - type: array - required: - - matchFeatures - type: object - type: array - matchFeatures: - description: MatchFeatures specifies a set of matcher terms - all of which must match. - items: - description: FeatureMatcherTerm defines requirements against - one feature set. All requirements (specified as MatchExpressions) - are evaluated against each element in the feature set. - properties: - feature: - type: string - matchExpressions: - additionalProperties: - description: "MatchExpression specifies an expression - to evaluate against a set of input values. It contains - an operator that is applied when matching the input - and an array of values that the operator evaluates - the input against. \n NB: CreateMatchExpression or - MustCreateMatchExpression() should be used for creating - new instances. NB: Validate() must be called if Op - or Value fields are modified or if a new instance - is created from scratch without using the helper functions." - properties: - op: - description: Op is the operator to be applied. - enum: - - In - - NotIn - - InRegexp - - Exists - - DoesNotExist - - Gt - - Lt - - GtLt - - IsTrue - - IsFalse - type: string - value: - description: Value is the list of values that the - operand evaluates the input against. Value should - be empty if the operator is Exists, DoesNotExist, - IsTrue or IsFalse. Value should contain exactly - one element if the operator is Gt or Lt and exactly - two elements if the operator is GtLt. In other - cases Value should contain at least one element. - items: - type: string - type: array - required: - - op - type: object - description: MatchExpressionSet contains a set of MatchExpressions, - each of which is evaluated against a set of input values. - type: object - required: - - feature - - matchExpressions - type: object - type: array - name: - description: Name of the rule. - type: string - vars: - additionalProperties: - type: string - description: Vars is the variables to store if the rule matches. - Variables do not directly inflict any changes in the node - object. However, they can be referenced from other rules enabling - more complex rule hierarchies, without exposing intermediary - output values as labels. - type: object - varsTemplate: - description: VarsTemplate specifies a template to expand for - dynamically generating multiple variables. Data (after template - expansion) must be keys with an optional value ([=]) - separated by newlines. - type: string - required: - - name - type: object - type: array - required: - - rules - type: object - required: - - spec - type: object - served: true - storage: true diff --git a/deployment/helm/node-feature-discovery/crds/nfd-api-crds.yaml b/deployment/helm/node-feature-discovery/crds/nfd-api-crds.yaml deleted file mode 100644 index c59796b836..0000000000 --- a/deployment/helm/node-feature-discovery/crds/nfd-api-crds.yaml +++ /dev/null @@ -1,334 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null - name: nodefeatures.nfd.k8s-sigs.io -spec: - group: nfd.k8s-sigs.io - names: - kind: NodeFeature - listKind: NodeFeatureList - plural: nodefeatures - singular: nodefeature - scope: Cluster - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: NodeFeature resource holds the features discovered for one node - in the cluster. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: NodeFeatureSpec describes a NodeFeature object. - properties: - domains: - description: Features is a list per-domain feature sets. This is the - full "raw" features data that nfd-worker has discovered. - items: - description: DomainFeatures is the collection of all discovered - features of one domain. - properties: - attributes: - additionalProperties: - description: AttributeFeatureSet is a set of features having - string value. - properties: - elements: - additionalProperties: - type: string - type: object - required: - - elements - type: object - type: object - flags: - additionalProperties: - description: FlagFeatureSet is a set of simple features only - containing names without values. - properties: - elements: - additionalProperties: - description: Nil is a dummy empty struct for protobuf - compatibility - type: object - type: object - required: - - elements - type: object - type: object - instances: - additionalProperties: - description: InstanceFeatureSet is a set of features each - of which is an instance having multiple attributes. - properties: - elements: - items: - description: InstanceFeature represents one instance - of a complex features, e.g. a device. - properties: - attributes: - additionalProperties: - type: string - type: object - required: - - attributes - type: object - type: array - required: - - elements - type: object - type: object - name: - type: string - required: - - attributes - - flags - - instances - - name - type: object - type: array - labelsRequest: - additionalProperties: - type: string - description: LabelsRequest is the set of node labels that are requested - to be generatd. - type: object - required: - - domains - type: object - required: - - spec - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null - name: nodefeaturerules.nfd.k8s-sigs.io -spec: - group: nfd.k8s-sigs.io - names: - kind: NodeFeatureRule - listKind: NodeFeatureRuleList - plural: nodefeaturerules - singular: nodefeaturerule - scope: Cluster - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: NodeFeatureRule resource specifies a configuration for feature-based - customization of node objects, such as node labeling. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: NodeFeatureRuleSpec describes a NodeFeatureRule. - properties: - rules: - description: Rules is a list of node customization rules. - items: - description: Rule defines a rule for node customization such as - labeling. - properties: - labels: - additionalProperties: - type: string - description: Labels to create if the rule matches. - type: object - labelsTemplate: - description: LabelsTemplate specifies a template to expand for - dynamically generating multiple labels. Data (after template - expansion) must be keys with an optional value ([=]) - separated by newlines. - type: string - matchAny: - description: MatchAny specifies a list of matchers one of which - must match. - items: - description: MatchAnyElem specifies one sub-matcher of MatchAny. - properties: - matchFeatures: - description: MatchFeatures specifies a set of matcher - terms all of which must match. - items: - description: FeatureMatcherTerm defines requirements - against one feature set. All requirements (specified - as MatchExpressions) are evaluated against each element - in the feature set. - properties: - feature: - type: string - matchExpressions: - additionalProperties: - description: "MatchExpression specifies an expression - to evaluate against a set of input values. It - contains an operator that is applied when matching - the input and an array of values that the operator - evaluates the input against. \n NB: CreateMatchExpression - or MustCreateMatchExpression() should be used - for creating new instances. NB: Validate() must - be called if Op or Value fields are modified - or if a new instance is created from scratch - without using the helper functions." - properties: - op: - description: Op is the operator to be applied. - enum: - - In - - NotIn - - InRegexp - - Exists - - DoesNotExist - - Gt - - Lt - - GtLt - - IsTrue - - IsFalse - type: string - value: - description: Value is the list of values that - the operand evaluates the input against. - Value should be empty if the operator is - Exists, DoesNotExist, IsTrue or IsFalse. - Value should contain exactly one element - if the operator is Gt or Lt and exactly - two elements if the operator is GtLt. In - other cases Value should contain at least - one element. - items: - type: string - type: array - required: - - op - type: object - description: MatchExpressionSet contains a set of - MatchExpressions, each of which is evaluated against - a set of input values. - type: object - required: - - feature - - matchExpressions - type: object - type: array - required: - - matchFeatures - type: object - type: array - matchFeatures: - description: MatchFeatures specifies a set of matcher terms - all of which must match. - items: - description: FeatureMatcherTerm defines requirements against - one feature set. All requirements (specified as MatchExpressions) - are evaluated against each element in the feature set. - properties: - feature: - type: string - matchExpressions: - additionalProperties: - description: "MatchExpression specifies an expression - to evaluate against a set of input values. It contains - an operator that is applied when matching the input - and an array of values that the operator evaluates - the input against. \n NB: CreateMatchExpression or - MustCreateMatchExpression() should be used for creating - new instances. NB: Validate() must be called if Op - or Value fields are modified or if a new instance - is created from scratch without using the helper functions." - properties: - op: - description: Op is the operator to be applied. - enum: - - In - - NotIn - - InRegexp - - Exists - - DoesNotExist - - Gt - - Lt - - GtLt - - IsTrue - - IsFalse - type: string - value: - description: Value is the list of values that the - operand evaluates the input against. Value should - be empty if the operator is Exists, DoesNotExist, - IsTrue or IsFalse. Value should contain exactly - one element if the operator is Gt or Lt and exactly - two elements if the operator is GtLt. In other - cases Value should contain at least one element. - items: - type: string - type: array - required: - - op - type: object - description: MatchExpressionSet contains a set of MatchExpressions, - each of which is evaluated against a set of input values. - type: object - required: - - feature - - matchExpressions - type: object - type: array - name: - description: Name of the rule. - type: string - vars: - additionalProperties: - type: string - description: Vars is the variables to store if the rule matches. - Variables do not directly inflict any changes in the node - object. However, they can be referenced from other rules enabling - more complex rule hierarchies, without exposing intermediary - output values as labels. - type: object - varsTemplate: - description: VarsTemplate specifies a template to expand for - dynamically generating multiple variables. Data (after template - expansion) must be keys with an optional value ([=]) - separated by newlines. - type: string - required: - - name - type: object - type: array - required: - - rules - type: object - required: - - spec - type: object - served: true - storage: true From 6364803b0c3df67b7f6562c9edee7dc58ec2eca8 Mon Sep 17 00:00:00 2001 From: Talor Itzhak Date: Thu, 24 Nov 2022 12:59:38 +0200 Subject: [PATCH 21/87] e2e: move pod utils to a seperate package By moving those utils in to a seperate package, we can make the functions names shorter and clearer. For example, instead of: ``` testutils.NFDWorkerPod(opts...) testutils.NFDMasterPod(opts...) testutils.SpecWithContainerImage(...) ``` we'll have: ``` testpod.NFDWorker(opts...) testpod.NFDMaster(opts...) testpod.SpecWithContainerImage(...) ``` It will also make the package more isolated and portable. Signed-off-by: Talor Itzhak --- test/e2e/node_feature_discovery.go | 42 +++++++++--------- test/e2e/topology_updater.go | 31 +++++++------- test/e2e/utils/{ => pod}/pod.go | 68 +++++++++++++++--------------- 3 files changed, 73 insertions(+), 68 deletions(-) rename test/e2e/utils/{ => pod}/pod.go (83%) diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index 93fcefdd2c..78ec5b8c3f 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -37,10 +37,12 @@ import ( e2elog "k8s.io/kubernetes/test/e2e/framework/log" e2enetwork "k8s.io/kubernetes/test/e2e/framework/network" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/pkg/apis/nfd/v1alpha1" nfdclient "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned" "sigs.k8s.io/node-feature-discovery/source/custom" testutils "sigs.k8s.io/node-feature-discovery/test/e2e/utils" + testpod "sigs.k8s.io/node-feature-discovery/test/e2e/utils/pod" ) var ( @@ -111,8 +113,8 @@ var _ = SIGDescribe("Node Feature Discovery", func() { // Launch nfd-master By("Creating nfd master pod and nfd-master service") - imageOpt := testutils.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)) - masterPod = f.PodClient().CreateSync(testutils.NFDMasterPod(imageOpt)) + imageOpt := testpod.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)) + masterPod = f.PodClient().CreateSync(testpod.NFDMaster(imageOpt)) // Create nfd-master service nfdSvc, err := testutils.CreateService(f.ClientSet, f.Namespace.Name) @@ -154,11 +156,11 @@ var _ = SIGDescribe("Node Feature Discovery", func() { // Launch nfd-worker By("Creating a nfd worker pod") - podSpecOpts := []testutils.PodSpecOption{ - testutils.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)), - testutils.SpecWithContainerExtraArgs("-oneshot", "-label-sources=fake"), + podSpecOpts := []testpod.SpecOption{ + testpod.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)), + testpod.SpecWithContainerExtraArgs("-oneshot", "-label-sources=fake"), } - workerPod := testutils.NFDWorkerPod(podSpecOpts...) + workerPod := testpod.NFDWorker(podSpecOpts...) workerPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), workerPod, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -204,13 +206,13 @@ var _ = SIGDescribe("Node Feature Discovery", func() { fConf := cfg.DefaultFeatures By("Creating nfd-worker daemonset") - podSpecOpts := []testutils.PodSpecOption{testutils.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag))} - workerDS := testutils.NFDWorkerDaemonSet(podSpecOpts...) + podSpecOpts := []testpod.SpecOption{testpod.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag))} + workerDS := testpod.NFDWorkerDaemonSet(podSpecOpts...) workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) By("Waiting for daemonset pods to be ready") - Expect(testutils.WaitForPodsReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred()) + Expect(testpod.WaitForReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred()) By("Getting node objects") nodeList, err := f.ClientSet.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) @@ -334,18 +336,18 @@ var _ = SIGDescribe("Node Feature Discovery", func() { Expect(err).NotTo(HaveOccurred()) By("Creating nfd-worker daemonset with configmap mounted") - podSpecOpts := []testutils.PodSpecOption{ - testutils.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)), - testutils.SpecWithConfigMap(cm1.Name, filepath.Join(custom.Directory, "cm1")), - testutils.SpecWithConfigMap(cm2.Name, filepath.Join(custom.Directory, "cm2")), + podSpecOpts := []testpod.SpecOption{ + testpod.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)), + testpod.SpecWithConfigMap(cm1.Name, filepath.Join(custom.Directory, "cm1")), + testpod.SpecWithConfigMap(cm2.Name, filepath.Join(custom.Directory, "cm2")), } - workerDS := testutils.NFDWorkerDaemonSet(podSpecOpts...) + workerDS := testpod.NFDWorkerDaemonSet(podSpecOpts...) workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) By("Waiting for daemonset pods to be ready") - Expect(testutils.WaitForPodsReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred()) + Expect(testpod.WaitForReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred()) By("Getting target node and checking labels") targetNode, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), targetNodeName, metav1.GetOptions{}) @@ -417,16 +419,16 @@ core: Expect(err).NotTo(HaveOccurred()) By("Creating nfd-worker daemonset") - podSpecOpts := []testutils.PodSpecOption{ - testutils.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)), - testutils.SpecWithConfigMap(cm.Name, "/etc/kubernetes/node-feature-discovery"), + podSpecOpts := []testpod.SpecOption{ + testpod.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)), + testpod.SpecWithConfigMap(cm.Name, "/etc/kubernetes/node-feature-discovery"), } - workerDS := testutils.NFDWorkerDaemonSet(podSpecOpts...) + workerDS := testpod.NFDWorkerDaemonSet(podSpecOpts...) workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) By("Waiting for daemonset pods to be ready") - Expect(testutils.WaitForPodsReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred()) + Expect(testpod.WaitForReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred()) expected := map[string]string{ "feature.node.kubernetes.io/e2e-flag-test-1": "true", diff --git a/test/e2e/topology_updater.go b/test/e2e/topology_updater.go index f8b1954f3e..d67cbf1851 100644 --- a/test/e2e/topology_updater.go +++ b/test/e2e/topology_updater.go @@ -39,6 +39,7 @@ import ( admissionapi "k8s.io/pod-security-admission/api" testutils "sigs.k8s.io/node-feature-discovery/test/e2e/utils" + testpod "sigs.k8s.io/node-feature-discovery/test/e2e/utils/pod" ) var _ = SIGDescribe("Node Feature Discovery topology updater", func() { @@ -71,8 +72,8 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { Expect(testutils.ConfigureRBAC(f.ClientSet, f.Namespace.Name)).NotTo(HaveOccurred()) - imageOpt := testutils.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)) - f.PodClient().CreateSync(testutils.NFDMasterPod(imageOpt)) + imageOpt := testpod.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)) + f.PodClient().CreateSync(testpod.NFDMaster(imageOpt)) // Create nfd-master service masterService, err := testutils.CreateService(f.ClientSet, f.Namespace.Name) @@ -86,7 +87,7 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { Expect(err).NotTo(HaveOccurred()) By("Waiting for daemonset pods to be ready") - Expect(testutils.WaitForPodsReady(f.ClientSet, f.Namespace.Name, topologyUpdaterDaemonSet.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred()) + Expect(testpod.WaitForReady(f.ClientSet, f.Namespace.Name, topologyUpdaterDaemonSet.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred()) label := labels.SelectorFromSet(map[string]string{"name": topologyUpdaterDaemonSet.Spec.Template.Labels["name"]}) pods, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).List(context.TODO(), metav1.ListOptions{LabelSelector: label.String()}) @@ -119,8 +120,8 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { kcfg := cfg.GetKubeletConfig() By(fmt.Sprintf("Using config (%#v)", kcfg)) - podSpecOpts := []testutils.PodSpecOption{testutils.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag))} - topologyUpdaterDaemonSet = testutils.NFDTopologyUpdaterDaemonSet(kcfg, podSpecOpts...) + podSpecOpts := []testpod.SpecOption{testpod.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag))} + topologyUpdaterDaemonSet = testpod.NFDTopologyUpdaterDaemonSet(kcfg, podSpecOpts...) }) It("should fill the node resource topologies CR with the data", func() { @@ -133,12 +134,12 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { By("getting the initial topology information") initialNodeTopo := testutils.GetNodeTopology(topologyClient, topologyUpdaterNode.Name) By("creating a pod consuming resources from the shared, non-exclusive CPU pool (best-effort QoS)") - sleeperPod := testutils.BestEffortSleeperPod() + sleeperPod := testpod.BestEffortSleeper() podMap := make(map[string]*corev1.Pod) pod := f.PodClient().CreateSync(sleeperPod) podMap[pod.Name] = pod - defer testutils.DeletePodsAsync(f, podMap) + defer testpod.DeleteAsync(f, podMap) cooldown := 30 * time.Second By(fmt.Sprintf("getting the updated topology - sleeping for %v", cooldown)) @@ -173,12 +174,12 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { By("getting the initial topology information") initialNodeTopo := testutils.GetNodeTopology(topologyClient, topologyUpdaterNode.Name) By("creating a pod consuming resources from the shared, non-exclusive CPU pool (guaranteed QoS, nonintegral request)") - sleeperPod := testutils.GuaranteedSleeperPod("500m") + sleeperPod := testpod.GuaranteedSleeper("500m") podMap := make(map[string]*corev1.Pod) pod := f.PodClient().CreateSync(sleeperPod) podMap[pod.Name] = pod - defer testutils.DeletePodsAsync(f, podMap) + defer testpod.DeleteAsync(f, podMap) cooldown := 30 * time.Second By(fmt.Sprintf("getting the updated topology - sleeping for %v", cooldown)) @@ -219,7 +220,7 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { By("getting the initial topology information") initialNodeTopo := testutils.GetNodeTopology(topologyClient, topologyUpdaterNode.Name) By("creating a pod consuming exclusive CPUs") - sleeperPod := testutils.GuaranteedSleeperPod("1000m") + sleeperPod := testpod.GuaranteedSleeper("1000m") // in case there is more than a single node in the cluster // we need to set the node name, so we'll have certainty about // which node we need to examine @@ -228,7 +229,7 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { podMap := make(map[string]*corev1.Pod) pod := f.PodClient().CreateSync(sleeperPod) podMap[pod.Name] = pod - defer testutils.DeletePodsAsync(f, podMap) + defer testpod.DeleteAsync(f, podMap) By("checking the changes in the updated topology") var finalNodeTopo *v1alpha1.NodeResourceTopology @@ -274,11 +275,11 @@ excludeList: kcfg := cfg.GetKubeletConfig() By(fmt.Sprintf("Using config (%#v)", kcfg)) - podSpecOpts := []testutils.PodSpecOption{ - testutils.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)), - testutils.SpecWithConfigMap(cm.Name, "/etc/kubernetes/node-feature-discovery"), + podSpecOpts := []testpod.SpecOption{ + testpod.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)), + testpod.SpecWithConfigMap(cm.Name, "/etc/kubernetes/node-feature-discovery"), } - topologyUpdaterDaemonSet = testutils.NFDTopologyUpdaterDaemonSet(kcfg, podSpecOpts...) + topologyUpdaterDaemonSet = testpod.NFDTopologyUpdaterDaemonSet(kcfg, podSpecOpts...) }) It("noderesourcetopology should not advertise the memory resource", func() { diff --git a/test/e2e/utils/pod.go b/test/e2e/utils/pod/pod.go similarity index 83% rename from test/e2e/utils/pod.go rename to test/e2e/utils/pod/pod.go index 6741f56b64..2751fa93f8 100644 --- a/test/e2e/utils/pod.go +++ b/test/e2e/utils/pod/pod.go @@ -1,5 +1,5 @@ /* -Copyright 2018-2022 The Kubernetes Authors. +Copyright 2022 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package utils +package pod import ( "context" @@ -35,6 +35,8 @@ import ( "k8s.io/kubectl/pkg/util/podutils" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/utils/pointer" + + "sigs.k8s.io/node-feature-discovery/test/e2e/utils" ) var pullIfNotPresent = flag.Bool("nfd.pull-if-not-present", false, "Pull Images if not present - not always") @@ -43,8 +45,8 @@ const ( PauseImage = "registry.k8s.io/pause" ) -// GuarenteedSleeperPod makes a Guaranteed QoS class Pod object which long enough forever but requires `cpuLimit` exclusive CPUs. -func GuaranteedSleeperPod(cpuLimit string) *corev1.Pod { +// GuaranteedSleeper makes a Guaranteed QoS class Pod object which long enough forever but requires `cpuLimit` exclusive CPUs. +func GuaranteedSleeper(cpuLimit string) *corev1.Pod { return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "sleeper-gu-pod", @@ -69,8 +71,8 @@ func GuaranteedSleeperPod(cpuLimit string) *corev1.Pod { } } -// BestEffortSleeperPod makes a Best Effort QoS class Pod object which sleeps long enough -func BestEffortSleeperPod() *corev1.Pod { +// BestEffortSleeper makes a Best Effort QoS class Pod object which sleeps long enough +func BestEffortSleeper() *corev1.Pod { return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "sleeper-be-pod", @@ -87,8 +89,8 @@ func BestEffortSleeperPod() *corev1.Pod { } } -// DeletePodsAsync concurrently deletes all the pods in the given name:pod_object mapping. Returns when the longer operation ends. -func DeletePodsAsync(f *framework.Framework, podMap map[string]*corev1.Pod) { +// DeleteAsync concurrently deletes all the pods in the given name:pod_object mapping. Returns when the longer operation ends. +func DeleteAsync(f *framework.Framework, podMap map[string]*corev1.Pod) { var wg sync.WaitGroup for _, pod := range podMap { wg.Add(1) @@ -96,14 +98,14 @@ func DeletePodsAsync(f *framework.Framework, podMap map[string]*corev1.Pod) { defer ginkgo.GinkgoRecover() defer wg.Done() - DeletePodSyncByName(f, podName) + DeleteSyncByName(f, podName) }(pod.Namespace, pod.Name) } wg.Wait() } -// DeletePodSyncByName deletes the pod identified by `podName` in the current namespace -func DeletePodSyncByName(f *framework.Framework, podName string) { +// DeleteSyncByName deletes the pod identified by `podName` in the current namespace +func DeleteSyncByName(f *framework.Framework, podName string) { gp := int64(0) delOpts := metav1.DeleteOptions{ GracePeriodSeconds: &gp, @@ -111,10 +113,10 @@ func DeletePodSyncByName(f *framework.Framework, podName string) { f.PodClient().DeleteSync(podName, delOpts, framework.DefaultPodDeletionTimeout) } -type PodSpecOption func(spec *corev1.PodSpec) +type SpecOption func(spec *corev1.PodSpec) -// NFDMasterPod provide NFD master pod definition -func NFDMasterPod(opts ...PodSpecOption) *corev1.Pod { +// NFDMaster provide NFD master pod definition +func NFDMaster(opts ...SpecOption) *corev1.Pod { yes := true no := false p := &corev1.Pod{ @@ -163,13 +165,13 @@ func NFDMasterPod(opts ...PodSpecOption) *corev1.Pod { return p } -// NFDWorkerPod provides NFD worker pod definition -func NFDWorkerPod(opts ...PodSpecOption) *corev1.Pod { +// NFDWorker provides NFD worker pod definition +func NFDWorker(opts ...SpecOption) *corev1.Pod { p := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "nfd-worker-" + string(uuid.NewUUID()), }, - Spec: *nfdWorkerPodSpec(opts...), + Spec: *nfdWorkerSpec(opts...), } p.Spec.RestartPolicy = corev1.RestartPolicyNever @@ -178,17 +180,17 @@ func NFDWorkerPod(opts ...PodSpecOption) *corev1.Pod { } // NFDWorkerDaemonSet provides the NFD daemon set worker definition -func NFDWorkerDaemonSet(opts ...PodSpecOption) *appsv1.DaemonSet { - return newDaemonSet("nfd-worker", nfdWorkerPodSpec(opts...)) +func NFDWorkerDaemonSet(opts ...SpecOption) *appsv1.DaemonSet { + return newDaemonSet("nfd-worker", nfdWorkerSpec(opts...)) } // NFDTopologyUpdaterDaemonSet provides the NFD daemon set topology updater -func NFDTopologyUpdaterDaemonSet(kc KubeletConfig, opts ...PodSpecOption) *appsv1.DaemonSet { - return newDaemonSet("nfd-topology-updater", nfdTopologyUpdaterPodSpec(kc, opts...)) +func NFDTopologyUpdaterDaemonSet(kc utils.KubeletConfig, opts ...SpecOption) *appsv1.DaemonSet { + return newDaemonSet("nfd-topology-updater", nfdTopologyUpdaterSpec(kc, opts...)) } -// SpecWithContainerImage returns a PodSpecOption that sets the image used by the first container. -func SpecWithContainerImage(image string) PodSpecOption { +// SpecWithContainerImage returns a SpecOption that sets the image used by the first container. +func SpecWithContainerImage(image string) SpecOption { return func(spec *corev1.PodSpec) { // NOTE: we might want to make the container number a parameter cnt := &spec.Containers[0] @@ -196,8 +198,8 @@ func SpecWithContainerImage(image string) PodSpecOption { } } -// SpecWithContainerExtraArgs returns a PodSpecOption that adds extra args to the first container. -func SpecWithContainerExtraArgs(args ...string) PodSpecOption { +// SpecWithContainerExtraArgs returns a SpecOption that adds extra args to the first container. +func SpecWithContainerExtraArgs(args ...string) SpecOption { return func(spec *corev1.PodSpec) { // NOTE: we might want to make the container number a parameter cnt := &spec.Containers[0] @@ -205,9 +207,9 @@ func SpecWithContainerExtraArgs(args ...string) PodSpecOption { } } -// SpecWithMasterNodeSelector returns a PodSpecOption that modifies the pod to +// SpecWithMasterNodeSelector returns a SpecOption that modifies the pod to // be run on a control plane node of the cluster. -func SpecWithMasterNodeSelector(args ...string) PodSpecOption { +func SpecWithMasterNodeSelector(args ...string) SpecOption { return func(spec *corev1.PodSpec) { spec.NodeSelector["node-role.kubernetes.io/control-plane"] = "" spec.Tolerations = append(spec.Tolerations, @@ -220,8 +222,8 @@ func SpecWithMasterNodeSelector(args ...string) PodSpecOption { } } -// SpecWithConfigMap returns a PodSpecOption that mounts a configmap to the first container. -func SpecWithConfigMap(name, mountPath string) PodSpecOption { +// SpecWithConfigMap returns a SpecOption that mounts a configmap to the first container. +func SpecWithConfigMap(name, mountPath string) SpecOption { return func(spec *corev1.PodSpec) { spec.Volumes = append(spec.Volumes, corev1.Volume{ @@ -265,7 +267,7 @@ func newDaemonSet(name string, podSpec *corev1.PodSpec) *appsv1.DaemonSet { } } -func nfdWorkerPodSpec(opts ...PodSpecOption) *corev1.PodSpec { +func nfdWorkerSpec(opts ...SpecOption) *corev1.PodSpec { yes := true no := false p := &corev1.PodSpec{ @@ -380,7 +382,7 @@ func nfdWorkerPodSpec(opts ...PodSpecOption) *corev1.PodSpec { return p } -func nfdTopologyUpdaterPodSpec(kc KubeletConfig, opts ...PodSpecOption) *corev1.PodSpec { +func nfdTopologyUpdaterSpec(kc utils.KubeletConfig, opts ...SpecOption) *corev1.PodSpec { p := &corev1.PodSpec{ Containers: []corev1.Container{ { @@ -472,10 +474,10 @@ func newHostPathType(typ corev1.HostPathType) *corev1.HostPathType { return hostPathType } -// WaitForPodsReady waits for the pods to become ready. +// WaitForReady waits for the pods to become ready. // NOTE: copied from k8s v1.22 after which is was removed from there. // Convenient for checking that all pods of a daemonset are ready. -func WaitForPodsReady(c clientset.Interface, ns, name string, minReadySeconds int) error { +func WaitForReady(c clientset.Interface, ns, name string, minReadySeconds int) error { const poll = 2 * time.Second label := labels.SelectorFromSet(labels.Set(map[string]string{"name": name})) options := metav1.ListOptions{LabelSelector: label.String()} From 9c725c378fbda2a85a40e110f656105bea8cfaf0 Mon Sep 17 00:00:00 2001 From: Talor Itzhak Date: Thu, 24 Nov 2022 13:23:34 +0200 Subject: [PATCH 22/87] e2e: separate daemonset functions from pod The new package should provide pod-related utilities, hence let's move all the daemonset-related utilities to their own package as well. Signed-off-by: Talor Itzhak --- test/e2e/node_feature_discovery.go | 7 ++-- test/e2e/topology_updater.go | 5 ++- test/e2e/utils/daemonset/daemonset.go | 58 +++++++++++++++++++++++++++ test/e2e/utils/pod/pod.go | 34 +--------------- 4 files changed, 66 insertions(+), 38 deletions(-) create mode 100644 test/e2e/utils/daemonset/daemonset.go diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index 78ec5b8c3f..8ea5647ca6 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -42,6 +42,7 @@ import ( nfdclient "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned" "sigs.k8s.io/node-feature-discovery/source/custom" testutils "sigs.k8s.io/node-feature-discovery/test/e2e/utils" + testds "sigs.k8s.io/node-feature-discovery/test/e2e/utils/daemonset" testpod "sigs.k8s.io/node-feature-discovery/test/e2e/utils/pod" ) @@ -207,7 +208,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { By("Creating nfd-worker daemonset") podSpecOpts := []testpod.SpecOption{testpod.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag))} - workerDS := testpod.NFDWorkerDaemonSet(podSpecOpts...) + workerDS := testds.NFDWorker(podSpecOpts...) workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -341,7 +342,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { testpod.SpecWithConfigMap(cm1.Name, filepath.Join(custom.Directory, "cm1")), testpod.SpecWithConfigMap(cm2.Name, filepath.Join(custom.Directory, "cm2")), } - workerDS := testpod.NFDWorkerDaemonSet(podSpecOpts...) + workerDS := testds.NFDWorker(podSpecOpts...) workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -423,7 +424,7 @@ core: testpod.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)), testpod.SpecWithConfigMap(cm.Name, "/etc/kubernetes/node-feature-discovery"), } - workerDS := testpod.NFDWorkerDaemonSet(podSpecOpts...) + workerDS := testds.NFDWorker(podSpecOpts...) workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) diff --git a/test/e2e/topology_updater.go b/test/e2e/topology_updater.go index d67cbf1851..043983a996 100644 --- a/test/e2e/topology_updater.go +++ b/test/e2e/topology_updater.go @@ -39,6 +39,7 @@ import ( admissionapi "k8s.io/pod-security-admission/api" testutils "sigs.k8s.io/node-feature-discovery/test/e2e/utils" + testds "sigs.k8s.io/node-feature-discovery/test/e2e/utils/daemonset" testpod "sigs.k8s.io/node-feature-discovery/test/e2e/utils/pod" ) @@ -121,7 +122,7 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { By(fmt.Sprintf("Using config (%#v)", kcfg)) podSpecOpts := []testpod.SpecOption{testpod.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag))} - topologyUpdaterDaemonSet = testpod.NFDTopologyUpdaterDaemonSet(kcfg, podSpecOpts...) + topologyUpdaterDaemonSet = testds.NFDTopologyUpdater(kcfg, podSpecOpts...) }) It("should fill the node resource topologies CR with the data", func() { @@ -279,7 +280,7 @@ excludeList: testpod.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)), testpod.SpecWithConfigMap(cm.Name, "/etc/kubernetes/node-feature-discovery"), } - topologyUpdaterDaemonSet = testpod.NFDTopologyUpdaterDaemonSet(kcfg, podSpecOpts...) + topologyUpdaterDaemonSet = testds.NFDTopologyUpdater(kcfg, podSpecOpts...) }) It("noderesourcetopology should not advertise the memory resource", func() { diff --git a/test/e2e/utils/daemonset/daemonset.go b/test/e2e/utils/daemonset/daemonset.go new file mode 100644 index 0000000000..920f0de0c9 --- /dev/null +++ b/test/e2e/utils/daemonset/daemonset.go @@ -0,0 +1,58 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package daemonset + +import ( + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" + + "sigs.k8s.io/node-feature-discovery/test/e2e/utils" + "sigs.k8s.io/node-feature-discovery/test/e2e/utils/pod" +) + +// NFDWorker provides the NFD daemon set worker definition +func NFDWorker(opts ...pod.SpecOption) *appsv1.DaemonSet { + return new("nfd-worker", &pod.NFDWorker(opts...).Spec) +} + +// NFDTopologyUpdater provides the NFD daemon set topology updater +func NFDTopologyUpdater(kc utils.KubeletConfig, opts ...pod.SpecOption) *appsv1.DaemonSet { + return new("nfd-topology-updater", pod.NFDTopologyUpdaterSpec(kc, opts...)) +} + +// new provide the new daemon set +func new(name string, podSpec *corev1.PodSpec) *appsv1.DaemonSet { + return &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name + "-" + string(uuid.NewUUID()), + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"name": name}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"name": name}, + }, + Spec: *podSpec, + }, + MinReadySeconds: 5, + }, + } +} diff --git a/test/e2e/utils/pod/pod.go b/test/e2e/utils/pod/pod.go index 2751fa93f8..75d39aeaa0 100644 --- a/test/e2e/utils/pod/pod.go +++ b/test/e2e/utils/pod/pod.go @@ -24,7 +24,6 @@ import ( "github.com/onsi/ginkgo/v2" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -179,16 +178,6 @@ func NFDWorker(opts ...SpecOption) *corev1.Pod { return p } -// NFDWorkerDaemonSet provides the NFD daemon set worker definition -func NFDWorkerDaemonSet(opts ...SpecOption) *appsv1.DaemonSet { - return newDaemonSet("nfd-worker", nfdWorkerSpec(opts...)) -} - -// NFDTopologyUpdaterDaemonSet provides the NFD daemon set topology updater -func NFDTopologyUpdaterDaemonSet(kc utils.KubeletConfig, opts ...SpecOption) *appsv1.DaemonSet { - return newDaemonSet("nfd-topology-updater", nfdTopologyUpdaterSpec(kc, opts...)) -} - // SpecWithContainerImage returns a SpecOption that sets the image used by the first container. func SpecWithContainerImage(image string) SpecOption { return func(spec *corev1.PodSpec) { @@ -246,27 +235,6 @@ func SpecWithConfigMap(name, mountPath string) SpecOption { } } -// newDaemonSet provide the new daemon set -func newDaemonSet(name string, podSpec *corev1.PodSpec) *appsv1.DaemonSet { - return &appsv1.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: name + "-" + string(uuid.NewUUID()), - }, - Spec: appsv1.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"name": name}, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"name": name}, - }, - Spec: *podSpec, - }, - MinReadySeconds: 5, - }, - } -} - func nfdWorkerSpec(opts ...SpecOption) *corev1.PodSpec { yes := true no := false @@ -382,7 +350,7 @@ func nfdWorkerSpec(opts ...SpecOption) *corev1.PodSpec { return p } -func nfdTopologyUpdaterSpec(kc utils.KubeletConfig, opts ...SpecOption) *corev1.PodSpec { +func NFDTopologyUpdaterSpec(kc utils.KubeletConfig, opts ...SpecOption) *corev1.PodSpec { p := &corev1.PodSpec{ Containers: []corev1.Container{ { From 0a065629306d538442316ba7340618354703f4e4 Mon Sep 17 00:00:00 2001 From: Talor Itzhak Date: Mon, 28 Nov 2022 20:22:56 +0200 Subject: [PATCH 23/87] e2e: simplify sleeper pod Make it more flexiable by allowing modifying both CPU and memory values, using functional options Signed-off-by: Talor Itzhak --- test/e2e/topology_updater.go | 15 +++++++++++++-- test/e2e/utils/pod/pod.go | 28 +++++++++++++++------------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/test/e2e/topology_updater.go b/test/e2e/topology_updater.go index 043983a996..05e2b65f26 100644 --- a/test/e2e/topology_updater.go +++ b/test/e2e/topology_updater.go @@ -19,6 +19,7 @@ package e2e import ( "context" "fmt" + "k8s.io/apimachinery/pkg/api/resource" "time" . "github.com/onsi/ginkgo/v2" @@ -175,7 +176,12 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { By("getting the initial topology information") initialNodeTopo := testutils.GetNodeTopology(topologyClient, topologyUpdaterNode.Name) By("creating a pod consuming resources from the shared, non-exclusive CPU pool (guaranteed QoS, nonintegral request)") - sleeperPod := testpod.GuaranteedSleeper("500m") + sleeperPod := testpod.GuaranteedSleeper(testpod.WithLimits( + corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + // any random reasonable amount is fine + corev1.ResourceMemory: resource.MustParse("100Mi"), + })) podMap := make(map[string]*corev1.Pod) pod := f.PodClient().CreateSync(sleeperPod) @@ -221,7 +227,12 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { By("getting the initial topology information") initialNodeTopo := testutils.GetNodeTopology(topologyClient, topologyUpdaterNode.Name) By("creating a pod consuming exclusive CPUs") - sleeperPod := testpod.GuaranteedSleeper("1000m") + sleeperPod := testpod.GuaranteedSleeper(testpod.WithLimits( + corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1000m"), + // any random reasonable amount is fine + corev1.ResourceMemory: resource.MustParse("100Mi"), + })) // in case there is more than a single node in the cluster // we need to set the node name, so we'll have certainty about // which node we need to examine diff --git a/test/e2e/utils/pod/pod.go b/test/e2e/utils/pod/pod.go index 75d39aeaa0..698d94b4d6 100644 --- a/test/e2e/utils/pod/pod.go +++ b/test/e2e/utils/pod/pod.go @@ -25,7 +25,6 @@ import ( "github.com/onsi/ginkgo/v2" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/uuid" @@ -45,8 +44,8 @@ const ( ) // GuaranteedSleeper makes a Guaranteed QoS class Pod object which long enough forever but requires `cpuLimit` exclusive CPUs. -func GuaranteedSleeper(cpuLimit string) *corev1.Pod { - return &corev1.Pod{ +func GuaranteedSleeper(opts ...func(pod *corev1.Pod)) *corev1.Pod { + p := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "sleeper-gu-pod", }, @@ -54,20 +53,23 @@ func GuaranteedSleeper(cpuLimit string) *corev1.Pod { RestartPolicy: corev1.RestartPolicyNever, Containers: []corev1.Container{ { - Name: "sleeper-gu-cnt", - Image: PauseImage, - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - // we use 1 core because that's the minimal meaningful quantity - corev1.ResourceName(corev1.ResourceCPU): resource.MustParse(cpuLimit), - // any random reasonable amount is fine - corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("100Mi"), - }, - }, + Name: "sleeper-gu-cnt", + Image: PauseImage, + Resources: corev1.ResourceRequirements{}, }, }, }, } + for _, o := range opts { + o(p) + } + return p +} + +func WithLimits(list corev1.ResourceList) func(p *corev1.Pod) { + return func(p *corev1.Pod) { + p.Spec.Containers[0].Resources.Limits = list + } } // BestEffortSleeper makes a Best Effort QoS class Pod object which sleeps long enough From cdc7558f6f7c5d9148c70de8e7c93b737163181b Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Thu, 1 Dec 2022 11:08:20 +0200 Subject: [PATCH 24/87] docs: better document custom resources Add a separate page for describing the custom resources used by NFD. Simplify the Introduction page by moving the details of NodeResourceTopology from there. Similarly, drop long NodeResourceTopology example from the quick-start page, making the page shorter and simpler. --- docs/get-started/introduction.md | 56 +++------------ docs/get-started/quick-start.md | 59 ---------------- docs/usage/custom-resources.md | 109 ++++++++++++++++++++++++++++++ docs/usage/customization-guide.md | 2 +- docs/usage/examples-and-demos.md | 2 +- 5 files changed, 121 insertions(+), 107 deletions(-) create mode 100644 docs/usage/custom-resources.md diff --git a/docs/get-started/introduction.md b/docs/get-started/introduction.md index 59ba8cd9de..f17f13bc5b 100644 --- a/docs/get-started/introduction.md +++ b/docs/get-started/introduction.md @@ -102,49 +102,13 @@ command line flag affects the annotation names Unapplicable annotations are not created, i.e. for example master.version is only created on nodes running nfd-master. -## NodeResourceTopology CR - -When run with NFD-Topology-Updater, NFD creates CR instances corresponding to -node resource hardware topology such as: - - ```yaml -apiVersion: topology.node.k8s.io/v1alpha1 -kind: NodeResourceTopology -metadata: - name: node1 -topologyPolicies: ["SingleNUMANodeContainerLevel"] -zones: - - name: node-0 - type: Node - resources: - - name: cpu - capacity: 20 - allocatable: 16 - available: 10 - - name: vendor/nic1 - capacity: 3 - allocatable: 3 - available: 3 - - name: node-1 - type: Node - resources: - - name: cpu - capacity: 30 - allocatable: 30 - available: 15 - - name: vendor/nic2 - capacity: 6 - allocatable: 6 - available: 6 - - name: node-2 - type: Node - resources: - - name: cpu - capacity: 30 - allocatable: 30 - available: 15 - - name: vendor/nic1 - capacity: 3 - allocatable: 3 - available: 3 - ``` +## Custom resources + +NFD takes use of some Kubernetes Custom Resources. + +NFD-Master uses [NodeFeatureRule](../usage/custom-resources/nodefeaturerule)s +for custom labeling of nodes. + +NFD-Topology-Updater creates +[NodeResourceTopology](../usage/custom-resources/noderesourcetopology) objects +that describe the hardware topology of node resources. diff --git a/docs/get-started/quick-start.md b/docs/get-started/quick-start.md index 57ba0103eb..c0400121b7 100644 --- a/docs/get-started/quick-start.md +++ b/docs/get-started/quick-start.md @@ -120,65 +120,6 @@ kind-control-plane 23s kind-worker 23s ``` -## Show the CR instances - -```bash -$ kubectl describe noderesourcetopologies.topology.node.k8s.io kind-control-plane -Name: kind-control-plane -Namespace: default -Labels: -Annotations: -API Version: topology.node.k8s.io/v1alpha1 -Kind: NodeResourceTopology -... -Topology Policies: - SingleNUMANodeContainerLevel -Zones: - Name: node-0 - Costs: - node-0: 10 - node-1: 20 - Resources: - Name: Cpu - Allocatable: 3 - Capacity: 3 - Available: 3 - Name: vendor/nic1 - Allocatable: 2 - Capacity: 2 - Available: 2 - Name: vendor/nic2 - Allocatable: 2 - Capacity: 2 - Available: 2 - Type: Node - Name: node-1 - Costs: - node-0: 20 - node-1: 10 - Resources: - Name: Cpu - Allocatable: 4 - Capacity: 4 - Available: 4 - Name: vendor/nic1 - Allocatable: 2 - Capacity: 2 - Available: 2 - Name: vendor/nic2 - Allocatable: 2 - Capacity: 2 - Available: 2 - Type: Node -Events: -``` - -The CR instances created can be used to gain insight into the allocatable -resources along with the granularity of those resources at a per-zone level -(represented by node-0 and node-1 in the above example) or can be used by an -external entity (e.g. topology-aware scheduler plugin) to take an action based -on the gathered information. - [podresource-api]: https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins/#monitoring-device-plugin-resources [feature-gate]: https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates diff --git a/docs/usage/custom-resources.md b/docs/usage/custom-resources.md new file mode 100644 index 0000000000..5d5e472f2f --- /dev/null +++ b/docs/usage/custom-resources.md @@ -0,0 +1,109 @@ +--- +title: "CRDs" +layout: default +sort: 6 +--- + +# Custom Resources +{: .no_toc} + +## Table of contents +{: .no_toc .text-delta} + +1. TOC +{:toc} + +--- + +NFD uses some Kubernetes [custom resources][custom-resources]. + +## NodeFeatureRule + +NodeFeatureRule is an NFD-specific custom resource that is designed for +rule-based custom labeling of nodes. NFD-Master watches for NodeFeatureRule +objects in the cluster and labels nodes according to the rules within. Some use +cases are e.g. application specific labeling in a specific environments or +being distributed by hardware vendors to create specific labels for their +devices. + +```yaml +apiVersion: nfd.k8s-sigs.io/v1alpha1 +kind: NodeFeatureRule +metadata: + name: example-rule +spec: + rules: + - name: "example rule" + labels: + "example-custom-feature": "true" + # Label is created if all of the rules below match + matchFeatures: + # Match if "veth" kernel module is loaded + - feature: kernel.loadedmodule + matchExpressions: + veth: {op: Exists} + # Match if any PCI device with vendor 8086 exists in the system + - feature: pci.device + matchExpressions: + vendor: {op: In, value: ["8086"]} +``` + +See the +[Customization guide](customization-guide#node-feature-rule-custom-resource) +for full documentation of the NodeFeatureRule resource and its usage. + +## NodeResourceTopology + +When run with NFD-Topology-Updater, NFD creates NodeResourceTopology objects +corresponding to node resource hardware topology such as: + +```yaml +apiVersion: topology.node.k8s.io/v1alpha1 +kind: NodeResourceTopology +metadata: + name: node1 +topologyPolicies: ["SingleNUMANodeContainerLevel"] +zones: + - name: node-0 + type: Node + resources: + - name: cpu + capacity: 20 + allocatable: 16 + available: 10 + - name: vendor/nic1 + capacity: 3 + allocatable: 3 + available: 3 + - name: node-1 + type: Node + resources: + - name: cpu + capacity: 30 + allocatable: 30 + available: 15 + - name: vendor/nic2 + capacity: 6 + allocatable: 6 + available: 6 + - name: node-2 + type: Node + resources: + - name: cpu + capacity: 30 + allocatable: 30 + available: 15 + - name: vendor/nic1 + capacity: 3 + allocatable: 3 + available: 3 +``` + +The NodeResourceTopology objects created by NFD can be used to gain insight +into the allocatable resources along with the granularity of those resources at +a per-zone level (represented by node-0 and node-1 in the above example) or can +be used by an external entity (e.g. topology-aware scheduler plugin) to take an +action based on the gathered information. + + +[custom-resources]: https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/ diff --git a/docs/usage/customization-guide.md b/docs/usage/customization-guide.md index 640cf7ac24..c34b345251 100644 --- a/docs/usage/customization-guide.md +++ b/docs/usage/customization-guide.md @@ -1,7 +1,7 @@ --- title: "Customization guide" layout: default -sort: 6 +sort: 7 --- # Customization guide diff --git a/docs/usage/examples-and-demos.md b/docs/usage/examples-and-demos.md index 1a3c9c65cd..7d64926292 100644 --- a/docs/usage/examples-and-demos.md +++ b/docs/usage/examples-and-demos.md @@ -1,7 +1,7 @@ --- title: "Examples and demos" layout: default -sort: 7 +sort: 8 --- # Examples and demos From 8a4538403770ebb51492c8e40a81ec3ecb8b05e9 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Thu, 1 Dec 2022 10:24:16 +0200 Subject: [PATCH 25/87] docs: simplify quick-start page Move topology-updater deployment notes to the topology-updater usage page. Also, rework the plaintext and headings a bit. --- docs/get-started/quick-start.md | 28 ++++++---------------------- docs/usage/nfd-topology-updater.md | 17 ++++++++++++++++- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/docs/get-started/quick-start.md b/docs/get-started/quick-start.md index c0400121b7..b2c234200c 100644 --- a/docs/get-started/quick-start.md +++ b/docs/get-started/quick-start.md @@ -76,30 +76,18 @@ feature-dependent-pod 1/1 Running 0 23s 10.36.0.4 node-2 ## Additional Optional Installation Steps -In order to deploy nfd-master and nfd-topology-updater daemons -use `topologyupdater` overlay. +### Deploy nfd-topology-updater -Deploy with kustomize -- creates a new namespace, service and required RBAC -rules and nfd-master and nfd-topology-updater daemons. +In order to deploy nfd-master and nfd-topology-updater daemons +use `topologyupdater` kustomize overlay. ```bash kubectl apply -k https://github.com/kubernetes-sigs/node-feature-discovery/deployment/overlays/topologyupdater?ref={{ site.release }} ``` -**NOTE:** - -[PodResource API][podresource-api] is a prerequisite for nfd-topology-updater. - -Preceding Kubernetes v1.23, the `kubelet` must be started with the following flag: +### Verify nfd-topology-updater -`--feature-gates=KubeletPodResourcesGetAllocatable=true` - -Starting Kubernetes v1.23, the `GetAllocatableResources` is enabled by default -through `KubeletPodResourcesGetAllocatable` [feature gate][feature-gate]. - -## Verify - -Wait until NFD master and NFD topologyupdater are running. +Wait until NFD topologyupdater (and NFD master) are running. ```bash $ kubectl -n node-feature-discovery get ds,deploy @@ -111,7 +99,7 @@ deployment.apps/nfd-master 1/1 1 1 17s ``` -Check that the NodeResourceTopology CR instances are created +Check that the NodeResourceTopology objects are created ```bash $ kubectl get noderesourcetopologies.topology.node.k8s.io @@ -119,7 +107,3 @@ NAME AGE kind-control-plane 23s kind-worker 23s ``` - - -[podresource-api]: https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins/#monitoring-device-plugin-resources -[feature-gate]: https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates diff --git a/docs/usage/nfd-topology-updater.md b/docs/usage/nfd-topology-updater.md index 353b43d9a9..bb2ad7f1f6 100644 --- a/docs/usage/nfd-topology-updater.md +++ b/docs/usage/nfd-topology-updater.md @@ -25,6 +25,17 @@ option. The default sleep interval is set to 60s which is the value when no In addition, it can avoid examining specific allocated resources given a configuration of resources to exclude via [`-excludeList`](../reference/topology-updater-configuration-reference.md#excludelist) +## Deployment Notes + +Kubelet [PodResource API][podresource-api] is a prerequisite for +nfd-topology-updater to be able to run. + +Preceding Kubernetes v1.23, the `kubelet` must be started with +`--feature-gates=KubeletPodResourcesGetAllocatable=true`. + +Starting from Kubernetes v1.23, the `KubeletPodResourcesGetAllocatable` +[feature gate][feature-gate]. is enabled by default + ## Topology-Updater Configuration NFD-Topology-Updater supports configuration through a configuration file. The @@ -58,4 +69,8 @@ for more details. The (empty-by-default) [example config](https://github.com/kubernetes-sigs/node-feature-discovery/blob/{{site.release}}/deployment/components/topology-updater-config/nfd-topology-updater.conf.example) contains all available configuration options and can be used as a reference -for creating a configuration. \ No newline at end of file +for creating a configuration. + + +[podresource-api]: https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins/#monitoring-device-plugin-resources +[feature-gate]: https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates From 42ae2162847251e0ad1ca41cd7343f173b42fae5 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Thu, 1 Dec 2022 16:03:56 +0200 Subject: [PATCH 26/87] test/e2e: no pod restart policy of nfd-worker by default Fixes stricter API check on daemonset pod spec that started to cause e2e test failures. RestartPolicyNever that we previously set (by defaylt) isn't compatible with DaemonSets. --- test/e2e/node_feature_discovery.go | 1 + test/e2e/utils/pod/pod.go | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index 8ea5647ca6..a11a1f0e53 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -158,6 +158,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { // Launch nfd-worker By("Creating a nfd worker pod") podSpecOpts := []testpod.SpecOption{ + testpod.SpecWithRestartPolicy(corev1.RestartPolicyNever), testpod.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)), testpod.SpecWithContainerExtraArgs("-oneshot", "-label-sources=fake"), } diff --git a/test/e2e/utils/pod/pod.go b/test/e2e/utils/pod/pod.go index 698d94b4d6..82485569cc 100644 --- a/test/e2e/utils/pod/pod.go +++ b/test/e2e/utils/pod/pod.go @@ -174,12 +174,16 @@ func NFDWorker(opts ...SpecOption) *corev1.Pod { }, Spec: *nfdWorkerSpec(opts...), } - - p.Spec.RestartPolicy = corev1.RestartPolicyNever - return p } +// SpecWithRestartPolicy returns a SpecOption that sets the pod restart policy +func SpecWithRestartPolicy(restartpolicy corev1.RestartPolicy) SpecOption { + return func(spec *corev1.PodSpec) { + spec.RestartPolicy = restartpolicy + } +} + // SpecWithContainerImage returns a SpecOption that sets the image used by the first container. func SpecWithContainerImage(image string) SpecOption { return func(spec *corev1.PodSpec) { From 32b252147c0da0caeec11a828256a0b4884ebd34 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Thu, 1 Dec 2022 17:59:42 +0200 Subject: [PATCH 27/87] docs: small update to customization guide Add a reference to the label rule format in the NodeFeatureRule section. Also make it explicit in the beginning of Hooks section that hooks are deprecated. --- docs/usage/customization-guide.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/usage/customization-guide.md b/docs/usage/customization-guide.md index c34b345251..f6ad998a87 100644 --- a/docs/usage/customization-guide.md +++ b/docs/usage/customization-guide.md @@ -74,6 +74,9 @@ Now, on X86 platforms the feature label appears after doing `modprobe dummy` on a system and correspondingly the label is removed after `rmmod dummy`. Note a re-labeling delay up to the sleep-interval of nfd-worker (1 minute by default). +See [Label rule format](#label-rule-format) for detailed description of +available fields and how to write labeling rules. + ### NodeFeatureRule controller NFD-Master acts as the controller for `NodeFeatureRule` objects. It applies these @@ -135,7 +138,7 @@ on the nfd-master command line. ### Hooks -The `local` source executes hooks found in +**DEPRECATED** The `local` source executes hooks found in `/etc/kubernetes/node-feature-discovery/source.d/`. The hook files must be executable and they are supposed to print all discovered features in `stdout`. With ELF binaries static linking is recommended as the selection of system From 72e523f2775425c70cf2ec7f698bcd752584ce94 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Thu, 1 Dec 2022 20:57:21 +0200 Subject: [PATCH 28/87] scripts/mdlint: update mdlint to v0.12.0 --- docs/mdl-style.rb | 3 ++- scripts/test-infra/mdlint.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/mdl-style.rb b/docs/mdl-style.rb index 11d47d3a8b..04c7dbe416 100644 --- a/docs/mdl-style.rb +++ b/docs/mdl-style.rb @@ -5,5 +5,6 @@ # Exclude MD041 - First line in file should be a top level header exclude_rule 'MD041' rule 'MD013', :tables => false -rule 'MD013', :code_blocks => false +rule 'MD007', :indent => 2 +rule 'MD013', :ignore_code_blocks => true rule 'MD024', :allow_different_nesting => true diff --git a/scripts/test-infra/mdlint.sh b/scripts/test-infra/mdlint.sh index 9f156805a9..ff1504ad86 100755 --- a/scripts/test-infra/mdlint.sh +++ b/scripts/test-infra/mdlint.sh @@ -1,6 +1,6 @@ #!/bin/bash -e # Install mdl -gem install mdl -v 0.11.0 +gem install mdl -v 0.12.0 # Run verify steps find docs/ -path docs/vendor -prune -false -o -name '*.md' | xargs mdl -s docs/mdl-style.rb From c1bdcd9511bed3c234ca9277e4e87051523ffe1c Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Fri, 2 Dec 2022 14:50:35 +0200 Subject: [PATCH 29/87] helm: drop NodeFeatureRule CRD from templates Helm 3 can manage CRDs in a more user friendly way. In fact, this now causes deployment failure as Helm automatically tries to install the CRD from the "crds/" subdir, too. --- .../node-feature-discovery/templates/nodefeaturerule-crd.yaml | 3 --- deployment/helm/node-feature-discovery/values.yaml | 3 --- 2 files changed, 6 deletions(-) delete mode 100644 deployment/helm/node-feature-discovery/templates/nodefeaturerule-crd.yaml diff --git a/deployment/helm/node-feature-discovery/templates/nodefeaturerule-crd.yaml b/deployment/helm/node-feature-discovery/templates/nodefeaturerule-crd.yaml deleted file mode 100644 index c02c633d89..0000000000 --- a/deployment/helm/node-feature-discovery/templates/nodefeaturerule-crd.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if .Values.nodeFeatureRule.createCRD }} -{{ .Files.Get "crds/nodefeaturerule-crd.yaml" }} -{{- end}} diff --git a/deployment/helm/node-feature-discovery/values.yaml b/deployment/helm/node-feature-discovery/values.yaml index 477f162975..2b3a5942aa 100644 --- a/deployment/helm/node-feature-discovery/values.yaml +++ b/deployment/helm/node-feature-discovery/values.yaml @@ -10,9 +10,6 @@ nameOverride: "" fullnameOverride: "" namespaceOverride: "" -nodeFeatureRule: - createCRD: true - master: instance: extraLabelNs: [] From 7840fe52e54d58de1169b58c7d15fa8f6e637ce4 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Fri, 2 Dec 2022 17:18:57 +0200 Subject: [PATCH 30/87] helm: fix mount name of topology-updater config --- .../helm/node-feature-discovery/templates/topologyupdater.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/helm/node-feature-discovery/templates/topologyupdater.yaml b/deployment/helm/node-feature-discovery/templates/topologyupdater.yaml index a53a6033c9..4ec86e6287 100644 --- a/deployment/helm/node-feature-discovery/templates/topologyupdater.yaml +++ b/deployment/helm/node-feature-discovery/templates/topologyupdater.yaml @@ -68,7 +68,7 @@ spec: mountPath: "/etc/kubernetes/node-feature-discovery/certs" readOnly: true {{- end }} - - name: topology-updater-conf + - name: nfd-topology-updater-conf mountPath: "/etc/kubernetes/node-feature-discovery" readOnly: true From 532e1193ce21c45dd3bf69becab74cb991a2d106 Mon Sep 17 00:00:00 2001 From: Feruzjon Muyassarov Date: Thu, 6 Oct 2022 17:04:04 +0300 Subject: [PATCH 31/87] Add taints field to NodeFeatureRule CR spec Extend NodeFeatureRule Spec with taints field to allow users to specify the list of the taints they want to be set on the node if rule matches. Signed-off-by: Feruzjon Muyassarov --- .../base/nfd-crds/nodefeaturerule-crd.yaml | 29 +++++++++++++++++++ .../crds/nodefeaturerule-crd.yaml | 29 +++++++++++++++++++ pkg/apis/nfd/v1alpha1/types.go | 5 ++++ .../nfd/v1alpha1/zz_generated.deepcopy.go | 8 +++++ 4 files changed, 71 insertions(+) diff --git a/deployment/base/nfd-crds/nodefeaturerule-crd.yaml b/deployment/base/nfd-crds/nodefeaturerule-crd.yaml index e8e6004d13..9a1fc53c83 100644 --- a/deployment/base/nfd-crds/nodefeaturerule-crd.yaml +++ b/deployment/base/nfd-crds/nodefeaturerule-crd.yaml @@ -189,6 +189,35 @@ spec: name: description: Name of the rule. type: string + taints: + description: Taints to create if the rule matches. + items: + description: The node this Taint is attached to has the "effect" + on any pod that does not tolerate the Taint. + properties: + effect: + description: Required. The effect of the taint on pods + that do not tolerate the taint. Valid effects are NoSchedule, + PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied to + a node. + type: string + timeAdded: + description: TimeAdded represents the time at which the + taint was added. It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint + key. + type: string + required: + - effect + - key + type: object + type: array vars: additionalProperties: type: string diff --git a/deployment/helm/node-feature-discovery/crds/nodefeaturerule-crd.yaml b/deployment/helm/node-feature-discovery/crds/nodefeaturerule-crd.yaml index e8e6004d13..9a1fc53c83 100644 --- a/deployment/helm/node-feature-discovery/crds/nodefeaturerule-crd.yaml +++ b/deployment/helm/node-feature-discovery/crds/nodefeaturerule-crd.yaml @@ -189,6 +189,35 @@ spec: name: description: Name of the rule. type: string + taints: + description: Taints to create if the rule matches. + items: + description: The node this Taint is attached to has the "effect" + on any pod that does not tolerate the Taint. + properties: + effect: + description: Required. The effect of the taint on pods + that do not tolerate the taint. Valid effects are NoSchedule, + PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied to + a node. + type: string + timeAdded: + description: TimeAdded represents the time at which the + taint was added. It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint + key. + type: string + required: + - effect + - key + type: object + type: array vars: additionalProperties: type: string diff --git a/pkg/apis/nfd/v1alpha1/types.go b/pkg/apis/nfd/v1alpha1/types.go index b3e4bd6027..c057cddf7e 100644 --- a/pkg/apis/nfd/v1alpha1/types.go +++ b/pkg/apis/nfd/v1alpha1/types.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha1 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -120,6 +121,10 @@ type Rule struct { // +optional VarsTemplate string `json:"varsTemplate"` + // Taints to create if the rule matches. + // +optional + Taints []corev1.Taint `json:"taints,omitempty"` + // MatchFeatures specifies a set of matcher terms all of which must match. // +optional MatchFeatures FeatureMatcher `json:"matchFeatures"` diff --git a/pkg/apis/nfd/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/nfd/v1alpha1/zz_generated.deepcopy.go index 1196b41394..47513f1335 100644 --- a/pkg/apis/nfd/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/nfd/v1alpha1/zz_generated.deepcopy.go @@ -6,6 +6,7 @@ package v1alpha1 import ( + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -438,6 +439,13 @@ func (in *Rule) DeepCopyInto(out *Rule) { (*out)[key] = val } } + if in.Taints != nil { + in, out := &in.Taints, &out.Taints + *out = make([]v1.Taint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.MatchFeatures != nil { in, out := &in.MatchFeatures, &out.MatchFeatures *out = make(FeatureMatcher, len(*in)) From 2bdf427b89e81e598d493e1a0ae8c4ce9db90407 Mon Sep 17 00:00:00 2001 From: Feruzjon Muyassarov Date: Thu, 6 Oct 2022 18:09:09 +0300 Subject: [PATCH 32/87] nfd-master logic update for setting node taints This commits extends NFD master code to support adding node taints from NodeFeatureRule CR. We also introduce a new annotation for taints which helps to identify if the taint set on node is owned by NFD or not. When user deletes the taint entry from NodeFeatureRule CR, NFD will remove the taint from the node. But to avoid accidental deletion of taints not owned by the NFD, it needs to know the owner. Keeping track of NFD set taints in the annotation can be used during the filtering of the owner. Also enable-taints flag is added to allow users opt in/out for node tainting feature. The flag takes precedence over taints defined in NodeFeatureRule CR. In other words, if enbale-taints is set to false(disabled) and user still defines taints on the CR, NFD will ignore those taints and skip them from setting on the node. Signed-off-by: Feruzjon Muyassarov --- cmd/nfd-master/main.go | 2 + pkg/apis/nfd/v1alpha1/annotations_labels.go | 3 + pkg/apis/nfd/v1alpha1/rule.go | 6 +- pkg/nfd-master/nfd-master.go | 127 ++++++++++++++++++-- 4 files changed, 123 insertions(+), 15 deletions(-) diff --git a/cmd/nfd-master/main.go b/cmd/nfd-master/main.go index 07c41c50f0..b18223563c 100644 --- a/cmd/nfd-master/main.go +++ b/cmd/nfd-master/main.go @@ -96,6 +96,8 @@ func initFlags(flagset *flag.FlagSet) *master.Args { "NB: the label namespace is omitted i.e. the filter is only applied to the name part after '/'.") flagset.BoolVar(&args.NoPublish, "no-publish", false, "Do not publish feature labels") + flagset.BoolVar(&args.EnableTaints, "enable-taints", false, + "Enable node tainting feature") flagset.BoolVar(&args.FeatureRulesController, "featurerules-controller", true, "Enable controller for NodeFeatureRule objects. Generates node labels based on the rules in these CRs.") flagset.IntVar(&args.Port, "port", 8080, diff --git a/pkg/apis/nfd/v1alpha1/annotations_labels.go b/pkg/apis/nfd/v1alpha1/annotations_labels.go index 0dc82a6a9e..7e4cbee250 100644 --- a/pkg/apis/nfd/v1alpha1/annotations_labels.go +++ b/pkg/apis/nfd/v1alpha1/annotations_labels.go @@ -43,4 +43,7 @@ const ( // WorkerVersionAnnotation is the annotation that holds the version of nfd-worker running on the node WorkerVersionAnnotation = AnnotationNs + "/worker.version" + + // NodeTaintsAnnotation is the annotation that holds the taints that nfd-master set on the node + NodeTaintsAnnotation = AnnotationNs + "/taints" ) diff --git a/pkg/apis/nfd/v1alpha1/rule.go b/pkg/apis/nfd/v1alpha1/rule.go index c352b62d7f..d7e3cd20cb 100644 --- a/pkg/apis/nfd/v1alpha1/rule.go +++ b/pkg/apis/nfd/v1alpha1/rule.go @@ -22,8 +22,8 @@ import ( "strings" "text/template" + corev1 "k8s.io/api/core/v1" "k8s.io/klog/v2" - "sigs.k8s.io/node-feature-discovery/pkg/utils" ) @@ -32,6 +32,7 @@ import ( type RuleOutput struct { Labels map[string]string Vars map[string]string + Taints []corev1.Taint } // Execute the rule against a set of input features. @@ -94,9 +95,8 @@ func (r *Rule) Execute(features *Features) (RuleOutput, error) { vars[k] = v } - ret := RuleOutput{Labels: labels, Vars: vars} + ret := RuleOutput{Labels: labels, Vars: vars, Taints: r.Taints} utils.KlogDump(2, fmt.Sprintf("rule %q matched with: ", r.Name), " ", ret) - return ret, nil } diff --git a/pkg/nfd-master/nfd-master.go b/pkg/nfd-master/nfd-master.go index 5111d57730..d46f16bd32 100644 --- a/pkg/nfd-master/nfd-master.go +++ b/pkg/nfd-master/nfd-master.go @@ -39,10 +39,12 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" + label "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/klog/v2" + controller "k8s.io/kubernetes/pkg/controller" + taintutils "k8s.io/kubernetes/pkg/util/taints" "sigs.k8s.io/node-feature-discovery/pkg/apihelper" nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/pkg/apis/nfd/v1alpha1" @@ -72,6 +74,7 @@ type Args struct { LabelWhiteList utils.RegexpVal FeatureRulesController bool NoPublish bool + EnableTaints bool Port int Prune bool VerifyNodeName bool @@ -294,6 +297,13 @@ func (m *nfdMaster) prune() error { return fmt.Errorf("failed to prune labels from node %q: %v", node.Name, err) } + // Prune taints + err = m.setTaints(cli, []corev1.Taint{}, node.Name) + + if err != nil { + return fmt.Errorf("failed to prune taints from node %q: %v", node.Name, err) + } + // Prune annotations node, err := m.apihelper.GetNode(cli, node.Name) if err != nil { @@ -392,14 +402,13 @@ func verifyNodeName(cert *x509.Certificate, nodeName string) error { err := cert.VerifyHostname(nodeName) if err != nil { - return fmt.Errorf("Certificate %q not valid for node %q: %v", cert.Subject.CommonName, nodeName, err) + return fmt.Errorf("certificate %q not valid for node %q: %v", cert.Subject.CommonName, nodeName, err) } return nil } // SetLabels implements LabelerServer func (m *nfdMaster) SetLabels(c context.Context, r *pb.SetLabelsRequest) (*pb.SetLabelsReply, error) { - err := authorizeClient(c, m.args.VerifyNodeName, r.NodeName) if err != nil { return &pb.SetLabelsReply{}, err @@ -420,7 +429,9 @@ func (m *nfdMaster) SetLabels(c context.Context, r *pb.SetLabelsRequest) (*pb.Se // NOTE: we effectively mangle the request struct by not creating a deep copy of the map rawLabels = r.Labels } - for k, v := range m.crLabels(r) { + crLabels, crTaints := m.processNodeFeatureRule(r) + + for k, v := range crLabels { rawLabels[k] = v } @@ -440,10 +451,101 @@ func (m *nfdMaster) SetLabels(c context.Context, r *pb.SetLabelsRequest) (*pb.Se klog.Errorf("failed to advertise labels: %v", err) return &pb.SetLabelsReply{}, err } + + // set taints + var taints []corev1.Taint + if m.args.EnableTaints { + taints = crTaints + } + + // Call setTaints even though the feature flag is disabled. This + // ensures that we delete NFD owned stale taints when flag got + // turned off. + err = m.setTaints(cli, taints, r.NodeName) + if err != nil { + return &pb.SetLabelsReply{}, err + } } return &pb.SetLabelsReply{}, nil } +// setTaints sets node taints and annotations based on the taints passed via +// nodeFeatureRule custom resorce. If empty list of taints is passed, currently +// NFD owned taints and annotations are removed from the node. +func (m *nfdMaster) setTaints(cli *kubernetes.Clientset, taints []corev1.Taint, nodeName string) error { + // Fetch the node object. + node, err := m.apihelper.GetNode(cli, nodeName) + if err != nil { + return err + } + + // De-serialize the taints annotation into corev1.Taint type for comparision below. + oldTaints := []corev1.Taint{} + if val, ok := node.Annotations[nfdv1alpha1.NodeTaintsAnnotation]; ok { + sts := strings.Split(val, ",") + oldTaints, _, err = taintutils.ParseTaints(sts) + if err != nil { + return err + } + } + + // Delete old nfd-managed taints that are not found in the set of new taints. + taintsUpdated := false + newNode := node.DeepCopy() + for _, taintToRemove := range oldTaints { + if taintutils.TaintExists(taints, &taintToRemove) { + continue + } + + newTaints, removed := taintutils.DeleteTaint(newNode.Spec.Taints, &taintToRemove) + if !removed { + klog.V(1).Infof("taint %q already deleted from node", taintToRemove.ToString()) + } + taintsUpdated = taintsUpdated || removed + newNode.Spec.Taints = newTaints + } + + // Add new taints found in the set of new taints. + for _, taint := range taints { + var updated bool + newNode, updated, err = taintutils.AddOrUpdateTaint(newNode, &taint) + if err != nil { + return fmt.Errorf("failed to add %q taint on node %v", taint, node.Name) + } + taintsUpdated = taintsUpdated || updated + } + + if taintsUpdated { + err = controller.PatchNodeTaints(context.TODO(), cli, nodeName, node, newNode) + if err != nil { + return fmt.Errorf("failed to patch the node %v", node.Name) + } + klog.Infof("updated node %q taints", nodeName) + } + + // Update node annotation that holds the taints managed by us + newAnnotations := map[string]string{} + if len(taints) > 0 { + // Serialize the new taints into string and update the annotation + // with that string. + taintStrs := make([]string, 0, len(taints)) + for _, taint := range taints { + taintStrs = append(taintStrs, taint.ToString()) + } + newAnnotations[nfdv1alpha1.NodeTaintsAnnotation] = strings.Join(taintStrs, ",") + } + + patches := createPatches([]string{nfdv1alpha1.NodeTaintsAnnotation}, node.Annotations, newAnnotations, "/metadata/annotations") + if len(patches) > 0 { + err = m.apihelper.PatchNode(cli, node.Name, patches) + if err != nil { + return fmt.Errorf("error while patching node object: %v", err) + } + klog.V(1).Infof("patched node %q annotations for taints", nodeName) + } + return nil +} + func authorizeClient(c context.Context, checkNodeName bool, nodeName string) error { if checkNodeName { // Client authorization. @@ -493,20 +595,21 @@ func (m *nfdMaster) UpdateNodeTopology(c context.Context, r *topologypb.NodeTopo return &topologypb.NodeTopologyResponse{}, nil } -func (m *nfdMaster) crLabels(r *pb.SetLabelsRequest) map[string]string { +func (m *nfdMaster) processNodeFeatureRule(r *pb.SetLabelsRequest) (map[string]string, []corev1.Taint) { if m.nfdController == nil { - return nil + return nil, nil } - l := make(map[string]string) - ruleSpecs, err := m.nfdController.ruleLister.List(labels.Everything()) + labels := make(map[string]string) + var taints []corev1.Taint + ruleSpecs, err := m.nfdController.ruleLister.List(label.Everything()) sort.Slice(ruleSpecs, func(i, j int) bool { return ruleSpecs[i].Name < ruleSpecs[j].Name }) if err != nil { klog.Errorf("failed to list NodeFeatureRule resources: %v", err) - return nil + return nil, nil } // Helper struct for rule processing @@ -527,9 +630,9 @@ func (m *nfdMaster) crLabels(r *pb.SetLabelsRequest) map[string]string { klog.Errorf("failed to process Rule %q: %v", rule.Name, err) continue } - + taints = append(taints, ruleOut.Taints...) for k, v := range ruleOut.Labels { - l[k] = v + labels[k] = v } // Feed back rule output to features map for subsequent rules to match @@ -538,7 +641,7 @@ func (m *nfdMaster) crLabels(r *pb.SetLabelsRequest) map[string]string { } } - return l + return labels, taints } // updateNodeFeatures ensures the Kubernetes node object is up to date, From 984a3de19894cc3fb2ed65d2339335724ea498eb Mon Sep 17 00:00:00 2001 From: Feruzjon Muyassarov Date: Wed, 30 Nov 2022 00:36:32 +0200 Subject: [PATCH 33/87] Document tainting feature Signed-off-by: Feruzjon Muyassarov --- deployment/base/nfd-crds/cr-sample.yaml | 6 ++ .../reference/master-commandline-reference.md | 12 ++++ docs/usage/customization-guide.md | 61 ++++++++++++++++++- 3 files changed, 77 insertions(+), 2 deletions(-) diff --git a/deployment/base/nfd-crds/cr-sample.yaml b/deployment/base/nfd-crds/cr-sample.yaml index dad9405d08..539757209b 100644 --- a/deployment/base/nfd-crds/cr-sample.yaml +++ b/deployment/base/nfd-crds/cr-sample.yaml @@ -7,6 +7,12 @@ spec: # The following feature demonstrates the capabilities of the matchFeatures and # matchAny matchers. - name: "my feature rule" + taints: + - effect: PreferNoSchedule + key: "feature.node.kubernetes.io/special-node" + value: "true" + - effect: NoExecute + key: "feature.node.kubernetes.io/dedicated-node" labels: "my-complex-feature": "my-value" # matchFeatures implements a logical AND over feature matchers. diff --git a/docs/reference/master-commandline-reference.md b/docs/reference/master-commandline-reference.md index aed7378578..8542742d10 100644 --- a/docs/reference/master-commandline-reference.md +++ b/docs/reference/master-commandline-reference.md @@ -99,6 +99,18 @@ Example: nfd-master -cert-file=/opt/nfd/master.crt -key-file=/opt/nfd/master.key -ca-file=/opt/nfd/ca.crt ``` +### -enable-taints + +The `-enable-taints` flag enables/disables node tainting feature of NFD. + +Default: *false* + +Example: + +```bash +nfd-master -enable-taints=true +``` + ### -key-file The `-key-file` is one of the three flags (together with `-ca-file` and diff --git a/docs/usage/customization-guide.md b/docs/usage/customization-guide.md index f6ad998a87..a9dba14eb5 100644 --- a/docs/usage/customization-guide.md +++ b/docs/usage/customization-guide.md @@ -30,8 +30,8 @@ labeling: ## NodeFeatureRule custom resource `NodeFeatureRule` objects provide an easy way to create vendor or application -specific labels. It uses a flexible rule-based mechanism for creating labels -based on node feature. +specific labels and taints. It uses a flexible rule-based mechanism for creating +labels and optionally taints based on node features. ### A NodeFeatureRule example @@ -76,6 +76,54 @@ re-labeling delay up to the sleep-interval of nfd-worker (1 minute by default). See [Label rule format](#label-rule-format) for detailed description of available fields and how to write labeling rules. +### NodeFeatureRule tainting feature + +This feature is experimental. + +In some circumstances it is desirable keep nodes with specialized hardware away from +running general workload and instead leave them for workloads that need the specialized +hardware. One way to achieve it is to taint the nodes with the specialized hardware +and add corresponding toleration to pods that require the special hardware. NFD +offers node tainting functionality which is disabled by default. User can define +one or more custom taints via the `taints` field of the NodeFeatureRule CR. The +same rule-based mechanism is applied here and the NFD taints only rule matching nodes. + +To enable the tainting feature, `--enable-taints` flag needs to be set to `true`. +If the flag `--enable-taints` is set to `false` (i.e. disabled), taints defined in +the NodeFeatureRule CR have no effect and will be ignored by the NFD master. + +**NOTE**: Before enabling any taints, make sure to edit nfd-worker daemonset to +tolerate the taints to be created. Otherwise, already running pods that do not +tolerate the taint are evicted immediately from the node including the nfd-worker +pod. + +Example NodeFeatureRule with custom taints: + +```yaml +apiVersion: nfd.k8s-sigs.io/v1alpha1 +kind: NodeFeatureRule +metadata: + name: my-sample-rule-object +spec: + rules: + - name: "my sample taint rule" + taints: + - effect: PreferNoSchedule + key: "feature.node.kubernetes.io/special-node" + value: "true" + - effect: NoExecute + key: "feature.node.kubernetes.io/dedicated-node" + matchFeatures: + - feature: kernel.loadedmodule + matchExpressions: + dummy: {op: Exists} + - feature: kernel.config + matchExpressions: + X86: {op: In, value: ["y"]} +``` + +In this example, if the `my sample taint rule` rule is matched, `feature.node.kubernetes.io/pci-0300_1d0f.present=true:NoExecute` +and `feature.node.kubernetes.io/cpu-cpuid.ADX:NoExecute` taints are set on the node. ### NodeFeatureRule controller @@ -365,6 +413,15 @@ details. labels specified in the `labels` field will override anything originating from `labelsTemplate`. +### Taints + +*taints* is a list of taint entries and each entry can have `key`, `value` and `effect`, +where the `value` is optional. Effect could be `NoSchedule`, `PreferNoSchedule` +or `NoExecute`. To learn more about the meaning of these effects, check out k8s [documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/). + +**NOTE** taints field is not available for the custom rules of nfd-worker and only +for NodeFeatureRule objects. + #### Vars The `.vars` field is a map of values (key-value pairs) to store for subsequent From 153815fa56342f0ad12b0974eb293ff2a429d3d4 Mon Sep 17 00:00:00 2001 From: Tariq Ibrahim Date: Mon, 5 Dec 2022 17:45:26 -0800 Subject: [PATCH 34/87] nfd-master svc should select only nfd-master pods --- deployment/helm/node-feature-discovery/templates/service.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/deployment/helm/node-feature-discovery/templates/service.yaml b/deployment/helm/node-feature-discovery/templates/service.yaml index 95b6d10b44..6731ca43ae 100644 --- a/deployment/helm/node-feature-discovery/templates/service.yaml +++ b/deployment/helm/node-feature-discovery/templates/service.yaml @@ -15,3 +15,4 @@ spec: name: grpc selector: {{- include "node-feature-discovery.selectorLabels" . | nindent 4 }} + role: master From 0834ec5cbfb676823dae8d50e8f6a08c369284d9 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Wed, 7 Dec 2022 13:44:52 +0200 Subject: [PATCH 35/87] go.mod: update to klauspost/cpuid to v2.2.2 Support detection of Intel TME (Total Memory Encryption) plus AMXFP16 and PREFETCHI. --- docs/usage/features.md | 2 ++ go.mod | 6 +++--- go.sum | 7 ++++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/usage/features.md b/docs/usage/features.md index 24797e8aab..56b61478ae 100644 --- a/docs/usage/features.md +++ b/docs/usage/features.md @@ -79,6 +79,7 @@ configuration options for details. | AVXVNNI | AVX (VEX encoded) VNNI neural network instructions | AMXBF16 | Advanced Matrix Extension, tile multiplication operations on BFLOAT16 numbers | AMXINT8 | Advanced Matrix Extension, tile multiplication operations on 8-bit integers +| AMXFP16 | Advanced Matrix Extension, tile multiplication operations on FP16 numbers | AMXTILE | Advanced Matrix Extension, base tile architecture support | AVX512BF16 | AVX-512 BFLOAT16 instructions | AVX512BITALG | AVX-512 bit Algorithms @@ -99,6 +100,7 @@ configuration options for details. | ENQCMD | Enqueue Command | GFNI | Galois Field New Instructions | HYPERVISOR | Running under hypervisor +| PREFETCHI | PREFETCHIT0/1 instructions | VAES | AVX-512 vector AES instructions | VPCLMULQDQ | Carry-less multiplication quadword diff --git a/go.mod b/go.mod index 074e4dae81..facd889027 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/google/go-cmp v0.5.8 github.com/jaypipes/ghw v0.8.1-0.20210827132705-c7224150a17e github.com/k8stopologyawareschedwg/noderesourcetopology-api v0.0.12 - github.com/klauspost/cpuid/v2 v2.1.0 + github.com/klauspost/cpuid/v2 v2.2.2 github.com/onsi/ginkgo/v2 v2.3.1 github.com/onsi/gomega v1.22.1 github.com/smartystreets/assertions v1.2.0 @@ -28,6 +28,7 @@ require ( k8s.io/kubectl v0.25.3 k8s.io/kubelet v0.25.3 k8s.io/kubernetes v1.25.3 + k8s.io/pod-security-admission v0.0.0 k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed sigs.k8s.io/yaml v1.2.0 ) @@ -163,7 +164,7 @@ require ( golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect - golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect + golang.org/x/sys v0.3.0 // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect @@ -188,7 +189,6 @@ require ( k8s.io/kube-scheduler v0.0.0 // indirect k8s.io/legacy-cloud-providers v0.0.0 // indirect k8s.io/mount-utils v0.25.0 // indirect - k8s.io/pod-security-admission v0.0.0 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.33 // indirect sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect diff --git a/go.sum b/go.sum index 84056b4670..46f4a4219b 100644 --- a/go.sum +++ b/go.sum @@ -422,8 +422,8 @@ github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/cpuid/v2 v2.1.0 h1:eyi1Ad2aNJMW95zcSbmGg7Cg6cq3ADwLpMAP96d8rF0= -github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.2 h1:xPMwiykqNK9VK0NYC3+jTMYv9I6Vl3YdjZgPZKG3zO0= +github.com/klauspost/cpuid/v2 v2.2.2/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -910,8 +910,9 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= From 881ee13654efc116d588368cb5a6104b4fa728e8 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Wed, 7 Dec 2022 16:25:43 +0200 Subject: [PATCH 36/87] docs: remove non-existent nodeFeatureRule.createCRD parameter This value was recently dropped. --- docs/deployment/helm.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/deployment/helm.md b/docs/deployment/helm.md index ef58213322..e2f0435cd3 100644 --- a/docs/deployment/helm.md +++ b/docs/deployment/helm.md @@ -95,7 +95,6 @@ We have introduced the following Chart parameters. | `imagePullSecrets` | list | [] | ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. If specified, these secrets will be passed to individual puller implementations for them to use. For example, in the case of docker, only DockerConfig type secrets are honored. [More info](https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod) | | `nameOverride` | string | | Override the name of the chart | | `fullnameOverride` | string | | Override a default fully qualified app name | -| `nodeFeatureRule.createCRD` | bool | true | Specifies whether to create the NodeFeatureRule CRD | | `tls.enable` | bool | false | Specifies whether to use TLS for communications between components | | `tls.certManager` | bool | false | If enabled, requires [cert-manager](https://cert-manager.io/docs/) to be installed and will automatically create the required TLS certificates | From 87b92f88ca1c8489ab9f649683d8aa31549dab2d Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Wed, 24 Aug 2022 12:01:38 +0300 Subject: [PATCH 37/87] nfd-worker: detect the namespace it is running in Implement detection of kubernetes namespace by reading file /var/run/secrets/kubernetes.io/serviceaccount/namespace Aa a fallback (if the file is not accessible) we take namespace from KUBERNETES_NAMESPACE environment variable. This is useful for e.g. testing and development where you might run nfd-worker directly from the command line on a host system. --- pkg/nfd-client/worker/nfd-worker.go | 24 +++++++++++--------- pkg/utils/kubernetes.go | 35 +++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 11 deletions(-) create mode 100644 pkg/utils/kubernetes.go diff --git a/pkg/nfd-client/worker/nfd-worker.go b/pkg/nfd-client/worker/nfd-worker.go index e7a6df73af..a7fdeb1f6e 100644 --- a/pkg/nfd-client/worker/nfd-worker.go +++ b/pkg/nfd-client/worker/nfd-worker.go @@ -95,14 +95,15 @@ type ConfigOverrideArgs struct { type nfdWorker struct { clientcommon.NfdBaseClient - args Args - certWatch *utils.FsWatcher - configFilePath string - config *NFDConfig - grpcClient pb.LabelerClient - stop chan struct{} // channel for signaling stop - featureSources []source.FeatureSource - labelSources []source.LabelSource + args Args + certWatch *utils.FsWatcher + configFilePath string + config *NFDConfig + kubernetesNamespace string + grpcClient pb.LabelerClient + stop chan struct{} // channel for signaling stop + featureSources []source.FeatureSource + labelSources []source.LabelSource } type duration struct { @@ -119,9 +120,10 @@ func NewNfdWorker(args *Args) (clientcommon.NfdClient, error) { nfd := &nfdWorker{ NfdBaseClient: base, - args: *args, - config: &NFDConfig{}, - stop: make(chan struct{}, 1), + args: *args, + config: &NFDConfig{}, + kubernetesNamespace: utils.GetKubernetesNamespace(), + stop: make(chan struct{}, 1), } if args.ConfigFile != "" { diff --git a/pkg/utils/kubernetes.go b/pkg/utils/kubernetes.go new file mode 100644 index 0000000000..4dd6ba09fd --- /dev/null +++ b/pkg/utils/kubernetes.go @@ -0,0 +1,35 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "os" + "strings" +) + +// GetKubernetesNamespace returns the kubernetes namespace we're running under, +// or an empty string if the namespace cannot be determined. +func GetKubernetesNamespace() string { + const kubernetesNamespaceFilePath = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" + if _, err := os.Stat(kubernetesNamespaceFilePath); err == nil { + data, err := os.ReadFile(kubernetesNamespaceFilePath) + if err == nil { + return strings.TrimSpace(string(data)) + } + } + return os.Getenv("KUBERNETES_NAMESPACE") +} From f13ed2d91c8b50becf77272927824096eee26e3e Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Fri, 2 Dec 2022 11:47:56 +0200 Subject: [PATCH 38/87] nfd-topology-updater: update NodeResourceTopology objects directly Drop the gRPC communication to nfd-master and connect to the Kubernetes API server directly when updating NodeResourceTopology objects. Topology-updater already has connection to the API server for listing Pods so this is not that dramatic change. It also simplifies the code a lot as there is no need for the NFD gRPC client and no need for managing TLS certs/keys. This change aligns nfd-topology-updater with the future direction of nfd-worker where the gRPC API is being dropped and replaced by a CRD-based API. This patch also update deployment files and documentation to reflect this change. --- cmd/nfd-topology-updater/main.go | 15 +- cmd/nfd-topology-updater/main_test.go | 8 +- .../topologyupdater-clusterrole.yaml | 8 + deployment/base/rbac/master-clusterrole.yaml | 8 - .../topologyupdater-daemonset.yaml | 3 +- .../templates/clusterrole.yaml | 18 +- .../templates/topologyupdater.yaml | 1 - .../topologyupdater/kustomization.yaml | 2 - docs/deployment/kustomize.md | 15 +- docs/get-started/introduction.md | 6 +- docs/get-started/quick-start.md | 11 +- .../topology-updater-commandline-reference.md | 78 ---- docs/usage/nfd-topology-updater.md | 10 +- .../topology-updater/nfd-topology-updater.go | 143 +++---- .../nfd-topology-updater_test.go | 169 --------- pkg/nfd-master/nfd-master.go | 80 ---- pkg/topologyupdater/doc.go | 20 - .../mock_NodeTopologyClient.go | 61 --- pkg/topologyupdater/topology-updater.pb.go | 353 ------------------ pkg/topologyupdater/topology-updater.proto | 35 -- test/e2e/topology_updater.go | 17 +- test/e2e/utils/pod/pod.go | 3 +- test/e2e/utils/rbac.go | 18 +- 23 files changed, 95 insertions(+), 987 deletions(-) delete mode 100644 pkg/nfd-client/topology-updater/nfd-topology-updater_test.go delete mode 100644 pkg/topologyupdater/doc.go delete mode 100644 pkg/topologyupdater/mock_NodeTopologyClient.go delete mode 100644 pkg/topologyupdater/topology-updater.pb.go delete mode 100644 pkg/topologyupdater/topology-updater.proto diff --git a/cmd/nfd-topology-updater/main.go b/cmd/nfd-topology-updater/main.go index 0b9bef9e80..e6115b3ee1 100644 --- a/cmd/nfd-topology-updater/main.go +++ b/cmd/nfd-topology-updater/main.go @@ -93,10 +93,7 @@ func main() { klog.Infof("detected kubelet Topology Manager policy %q", tmPolicy) // Get new TopologyUpdater instance - instance, err := topology.NewTopologyUpdater(*args, *resourcemonitorArgs, tmPolicy) - if err != nil { - klog.Exitf("failed to initialize TopologyUpdater instance: %v", err) - } + instance := topology.NewTopologyUpdater(*args, *resourcemonitorArgs, tmPolicy) if err = instance.Run(); err != nil { klog.Exit(err) @@ -129,12 +126,6 @@ func initFlags(flagset *flag.FlagSet) (*topology.Args, *resourcemonitor.Args) { args := &topology.Args{} resourcemonitorArgs := &resourcemonitor.Args{} - flagset.StringVar(&args.CaFile, "ca-file", "", - "Root certificate for verifying connections") - flagset.StringVar(&args.CertFile, "cert-file", "", - "Certificate used for authenticating connections") - flagset.StringVar(&args.KeyFile, "key-file", "", - "Private key matching -cert-file") flagset.BoolVar(&args.Oneshot, "oneshot", false, "Update once and exit") flagset.BoolVar(&args.NoPublish, "no-publish", false, @@ -151,10 +142,6 @@ func initFlags(flagset *flag.FlagSet) (*topology.Args, *resourcemonitor.Args) { "API auth token file path. It is used to request kubelet configz endpoint, only takes effect when kubelet-config-uri is https. Default to /var/run/secrets/kubernetes.io/serviceaccount/token.") flagset.StringVar(&resourcemonitorArgs.PodResourceSocketPath, "podresources-socket", hostpath.VarDir.Path("lib/kubelet/pod-resources/kubelet.sock"), "Pod Resource Socket path to use.") - flagset.StringVar(&args.Server, "server", "localhost:8080", - "NFD server address to connecto to.") - flagset.StringVar(&args.ServerNameOverride, "server-name-override", "", - "Hostname expected from server certificate, useful in testing") flagset.StringVar(&args.ConfigFile, "config", "/etc/kubernetes/node-feature-discovery/nfd-topology-updater.conf", "Config file to use.") diff --git a/cmd/nfd-topology-updater/main_test.go b/cmd/nfd-topology-updater/main_test.go index dd2ba86e08..b7bdd06f3d 100644 --- a/cmd/nfd-topology-updater/main_test.go +++ b/cmd/nfd-topology-updater/main_test.go @@ -87,16 +87,10 @@ func TestArgsParse(t *testing.T) { "-no-publish", "-sleep-interval=30s", "-kubelet-config-uri=file:///path/testconfig.yaml", - "-podresources-socket=/path/testkubelet.sock", - "-ca-file=ca", - "-cert-file=crt", - "-key-file=key") + "-podresources-socket=/path/testkubelet.sock") Convey("-no-publish is set and args.sources is set to appropriate values", func() { So(args.NoPublish, ShouldBeTrue) - So(args.CaFile, ShouldEqual, "ca") - So(args.CertFile, ShouldEqual, "crt") - So(args.KeyFile, ShouldEqual, "key") So(finderArgs.SleepInterval, ShouldEqual, 30*time.Second) So(finderArgs.KubeletConfigURI, ShouldEqual, "file:///path/testconfig.yaml") So(finderArgs.PodResourceSocketPath, ShouldEqual, "/path/testkubelet.sock") diff --git a/deployment/base/rbac-topologyupdater/topologyupdater-clusterrole.yaml b/deployment/base/rbac-topologyupdater/topologyupdater-clusterrole.yaml index 1719edbe06..493f396c12 100644 --- a/deployment/base/rbac-topologyupdater/topologyupdater-clusterrole.yaml +++ b/deployment/base/rbac-topologyupdater/topologyupdater-clusterrole.yaml @@ -22,3 +22,11 @@ rules: - pods verbs: - get +- apiGroups: + - topology.node.k8s.io + resources: + - noderesourcetopologies + verbs: + - create + - get + - update diff --git a/deployment/base/rbac/master-clusterrole.yaml b/deployment/base/rbac/master-clusterrole.yaml index 975616ecf7..ca4b5705aa 100644 --- a/deployment/base/rbac/master-clusterrole.yaml +++ b/deployment/base/rbac/master-clusterrole.yaml @@ -12,14 +12,6 @@ rules: - patch - update - list -- apiGroups: - - topology.node.k8s.io - resources: - - noderesourcetopologies - verbs: - - create - - get - - update - apiGroups: - nfd.k8s-sigs.io resources: diff --git a/deployment/base/topologyupdater-daemonset/topologyupdater-daemonset.yaml b/deployment/base/topologyupdater-daemonset/topologyupdater-daemonset.yaml index b894831488..8971ea8abb 100644 --- a/deployment/base/topologyupdater-daemonset/topologyupdater-daemonset.yaml +++ b/deployment/base/topologyupdater-daemonset/topologyupdater-daemonset.yaml @@ -21,5 +21,4 @@ spec: imagePullPolicy: Always command: - "nfd-topology-updater" - args: - - "-server=nfd-master:8080" + args: [] diff --git a/deployment/helm/node-feature-discovery/templates/clusterrole.yaml b/deployment/helm/node-feature-discovery/templates/clusterrole.yaml index 587a5a9554..f1c5805a13 100644 --- a/deployment/helm/node-feature-discovery/templates/clusterrole.yaml +++ b/deployment/helm/node-feature-discovery/templates/clusterrole.yaml @@ -32,16 +32,6 @@ rules: - get - list - watch -{{- if .Values.topologyUpdater.enable }} -- apiGroups: - - topology.node.k8s.io - resources: - - noderesourcetopologies - verbs: - - create - - get - - update -{{- end }} {{- end }} --- @@ -66,4 +56,12 @@ rules: - pods verbs: - get +- apiGroups: + - topology.node.k8s.io + resources: + - noderesourcetopologies + verbs: + - create + - get + - update {{- end }} diff --git a/deployment/helm/node-feature-discovery/templates/topologyupdater.yaml b/deployment/helm/node-feature-discovery/templates/topologyupdater.yaml index a53a6033c9..5f35ca15df 100644 --- a/deployment/helm/node-feature-discovery/templates/topologyupdater.yaml +++ b/deployment/helm/node-feature-discovery/templates/topologyupdater.yaml @@ -40,7 +40,6 @@ spec: command: - "nfd-topology-updater" args: - - "--server={{ include "node-feature-discovery.fullname" . }}-master:{{ .Values.master.service.port }}" {{- if .Values.topologyUpdater.updateInterval | empty | not }} - "--sleep-interval={{ .Values.topologyUpdater.updateInterval }}" {{- else }} diff --git a/deployment/overlays/topologyupdater/kustomization.yaml b/deployment/overlays/topologyupdater/kustomization.yaml index e007b40a0d..b344d993b8 100644 --- a/deployment/overlays/topologyupdater/kustomization.yaml +++ b/deployment/overlays/topologyupdater/kustomization.yaml @@ -4,9 +4,7 @@ kind: Kustomization namespace: node-feature-discovery bases: -- ../../base/rbac - ../../base/rbac-topologyupdater -- ../../base/master - ../../base/noderesourcetopologies-crd - ../../base/topologyupdater-daemonset diff --git a/docs/deployment/kustomize.md b/docs/deployment/kustomize.md index 75fd504d9c..b5d84a0d74 100644 --- a/docs/deployment/kustomize.md +++ b/docs/deployment/kustomize.md @@ -101,9 +101,9 @@ node(s) will run extra job instance(s) to satisfy the request. ### Master Worker Topologyupdater -NFD Master, NFD worker and NFD Topologyupdater can be configured to be deployed -as separate pods. The `master-worker-topologyupdater` overlay may be used to -achieve this: +NFD-Master, nfd-worker and nfd-topology-updater can be configured to be +deployed as separate pods. The `master-worker-topologyupdater` overlay may be +used to achieve this: ```bash kubectl apply -k https://github.com/kubernetes-sigs/node-feature-discovery/deployment/overlays/master-worker-topologyupdater?ref={{ site.release }} @@ -112,7 +112,7 @@ kubectl apply -k https://github.com/kubernetes-sigs/node-feature-discovery/deplo ### Topologyupdater -In order to deploy just NFD master and NFD Topologyupdater (without nfd-worker) +In order to deploy just nfd-topology-updater (without nfd-master and nfd-worker) use the `topologyupdater` overlay: ```bash @@ -120,10 +120,9 @@ kubectl apply -k https://github.com/kubernetes-sigs/node-feature-discovery/deplo ``` -NFD Topologyupdater can be configured along with the `default` overlay -(which deploys NFD worker and NFD master) where all the software components -are deployed as separate pods. The `topologyupdater` overlay may be used -along with `default` overlay to achieve this: +NFD-Topology-Updater can be configured along with the `default` overlay +(which deploys nfd-worker and nfd-master) where all the software components +are deployed as separate pods; ```bash diff --git a/docs/get-started/introduction.md b/docs/get-started/introduction.md index f17f13bc5b..3c6cac09a7 100644 --- a/docs/get-started/introduction.md +++ b/docs/get-started/introduction.md @@ -42,9 +42,9 @@ instance of nfd-worker is supposed to be running on each node of the cluster, NFD-Topology-Updater is a daemon responsible for examining allocated resources on a worker node to account for resources available to be allocated to new pod on a per-zone basis (where a zone can be a NUMA node). It then -communicates the information to nfd-master which does the -[NodeResourceTopology CR](#noderesourcetopology-cr) creation corresponding -to all the nodes in the cluster. One instance of nfd-topology-updater is +creates or updates a +[NodeResourceTopology](../usage/custom-resources#noderesourcetopology) custom +resource object specific to this node. One instance of nfd-topology-updater is supposed to be running on each node of the cluster. ## Feature Discovery diff --git a/docs/get-started/quick-start.md b/docs/get-started/quick-start.md index b2c234200c..b169ca9c92 100644 --- a/docs/get-started/quick-start.md +++ b/docs/get-started/quick-start.md @@ -78,8 +78,8 @@ feature-dependent-pod 1/1 Running 0 23s 10.36.0.4 node-2 ### Deploy nfd-topology-updater -In order to deploy nfd-master and nfd-topology-updater daemons -use `topologyupdater` kustomize overlay. +In order to deploy nfd-topology-updater use the `topologyupdater` kustomize +overlay. ```bash kubectl apply -k https://github.com/kubernetes-sigs/node-feature-discovery/deployment/overlays/topologyupdater?ref={{ site.release }} @@ -87,16 +87,13 @@ kubectl apply -k https://github.com/kubernetes-sigs/node-feature-discovery/deplo ### Verify nfd-topology-updater -Wait until NFD topologyupdater (and NFD master) are running. +Wait until nfd-topology-updater is running. ```bash -$ kubectl -n node-feature-discovery get ds,deploy +$ kubectl -n node-feature-discovery get ds NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE daemonset.apps/nfd-topology-updater 2 2 2 2 2 5s -NAME READY UP-TO-DATE AVAILABLE AGE -deployment.apps/nfd-master 1/1 1 1 17s - ``` Check that the NodeResourceTopology objects are created diff --git a/docs/reference/topology-updater-commandline-reference.md b/docs/reference/topology-updater-commandline-reference.md index 197d950421..101824a676 100644 --- a/docs/reference/topology-updater-commandline-reference.md +++ b/docs/reference/topology-updater-commandline-reference.md @@ -46,84 +46,6 @@ Example: nfd-topology-updater -config=/opt/nfd/nfd-topology-updater.conf ``` -### -server - -The `-server` flag specifies the address of the nfd-master endpoint where to -connect to. - -Default: localhost:8080 - -Example: - -```bash -nfd-topology-updater -server=nfd-master.nfd.svc.cluster.local:443 -``` - -### -ca-file - -The `-ca-file` is one of the three flags (together with `-cert-file` and -`-key-file`) controlling the mutual TLS authentication on the topology-updater side. -This flag specifies the TLS root certificate that is used for verifying the -authenticity of nfd-master. - -Default: *empty* - -Note: Must be specified together with `-cert-file` and `-key-file` - -Example: - -```bash -nfd-topology-updater -ca-file=/opt/nfd/ca.crt -cert-file=/opt/nfd/updater.crt -key-file=/opt/nfd/updater.key -``` - -### -cert-file - -The `-cert-file` is one of the three flags (together with `-ca-file` and -`-key-file`) controlling mutual TLS authentication on the topology-updater -side. This flag specifies the TLS certificate presented for authenticating -outgoing requests. - -Default: *empty* - -Note: Must be specified together with `-ca-file` and `-key-file` - -Example: - -```bash -nfd-topology-updater -cert-file=/opt/nfd/updater.crt -key-file=/opt/nfd/updater.key -ca-file=/opt/nfd/ca.crt -``` - -### -key-file - -The `-key-file` is one of the three flags (together with `-ca-file` and -`-cert-file`) controlling the mutual TLS authentication on topology-updater -side. This flag specifies the private key corresponding the given certificate file -(`-cert-file`) that is used for authenticating outgoing requests. - -Default: *empty* - -Note: Must be specified together with `-cert-file` and `-ca-file` - -Example: - -```bash -nfd-topology-updater -key-file=/opt/nfd/updater.key -cert-file=/opt/nfd/updater.crt -ca-file=/opt/nfd/ca.crt -``` - -### -server-name-override - -The `-server-name-override` flag specifies the common name (CN) which to -expect from the nfd-master TLS certificate. This flag is mostly intended for -development and debugging purposes. - -Default: *empty* - -Example: - -```bash -nfd-topology-updater -server-name-override=localhost -``` - ### -no-publish The `-no-publish` flag disables all communication with the nfd-master, making diff --git a/docs/usage/nfd-topology-updater.md b/docs/usage/nfd-topology-updater.md index bb2ad7f1f6..c4a6b1be19 100644 --- a/docs/usage/nfd-topology-updater.md +++ b/docs/usage/nfd-topology-updater.md @@ -10,11 +10,11 @@ sort: 5 --- NFD-Topology-Updater is preferably run as a Kubernetes DaemonSet. This assures -re-examination (and CR updates) on regular intervals capturing changes in -the allocated resources and hence the allocatable resources on a per zone -basis. It makes sure that more CR instances are created as new nodes get -added to the cluster. Topology-Updater connects to the nfd-master service -to create CR instances corresponding to nodes. +re-examination on regular intervals, capturing changes in the allocated +resources and hence the allocatable resources on a per zone basis by updating +[NodeResourceTopology](custom-resources#noderesourcetopology) custom resources. +It makes sure that new NodeResourceTopology instances are created for each new +nodes that get added to the cluster. When run as a daemonset, nodes are re-examined for the allocated resources (to determine the information of the allocatable resources on a per zone basis diff --git a/pkg/nfd-client/topology-updater/nfd-topology-updater.go b/pkg/nfd-client/topology-updater/nfd-topology-updater.go index 1853536e20..7ada9cb3ba 100644 --- a/pkg/nfd-client/topology-updater/nfd-topology-updater.go +++ b/pkg/nfd-client/topology-updater/nfd-topology-updater.go @@ -22,16 +22,16 @@ import ( "path/filepath" "time" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/klog/v2" v1alpha1 "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/apis/topology/v1alpha1" "golang.org/x/net/context" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/node-feature-discovery/pkg/apihelper" - nfdclient "sigs.k8s.io/node-feature-discovery/pkg/nfd-client" "sigs.k8s.io/node-feature-discovery/pkg/podres" "sigs.k8s.io/node-feature-discovery/pkg/resourcemonitor" - pb "sigs.k8s.io/node-feature-discovery/pkg/topologyupdater" "sigs.k8s.io/node-feature-discovery/pkg/utils" "sigs.k8s.io/node-feature-discovery/pkg/version" "sigs.k8s.io/yaml" @@ -39,11 +39,12 @@ import ( // Args are the command line arguments type Args struct { - nfdclient.Args NoPublish bool Oneshot bool KubeConfigFile string ConfigFile string + + Klog map[string]*utils.KlogFlagVal } // NFDConfig contains the configuration settings of NFDTopologyUpdater. @@ -52,38 +53,32 @@ type NFDConfig struct { } type NfdTopologyUpdater interface { - nfdclient.NfdClient - Update(v1alpha1.ZoneList) error + Run() error + Stop() } type staticNodeInfo struct { + nodeName string tmPolicy string } type nfdTopologyUpdater struct { - nfdclient.NfdBaseClient nodeInfo *staticNodeInfo args Args + apihelper apihelper.APIHelpers resourcemonitorArgs resourcemonitor.Args - certWatch *utils.FsWatcher - client pb.NodeTopologyClient stop chan struct{} // channel for signaling stop configFilePath string config *NFDConfig } // NewTopologyUpdater creates a new NfdTopologyUpdater instance. -func NewTopologyUpdater(args Args, resourcemonitorArgs resourcemonitor.Args, policy string) (NfdTopologyUpdater, error) { - base, err := nfdclient.NewNfdBaseClient(&args.Args) - if err != nil { - return nil, err - } - +func NewTopologyUpdater(args Args, resourcemonitorArgs resourcemonitor.Args, policy string) NfdTopologyUpdater { nfd := &nfdTopologyUpdater{ - NfdBaseClient: base, args: args, resourcemonitorArgs: resourcemonitorArgs, nodeInfo: &staticNodeInfo{ + nodeName: os.Getenv("NODE_NAME"), tmPolicy: policy, }, stop: make(chan struct{}, 1), @@ -92,27 +87,26 @@ func NewTopologyUpdater(args Args, resourcemonitorArgs resourcemonitor.Args, pol if args.ConfigFile != "" { nfd.configFilePath = filepath.Clean(args.ConfigFile) } - return nfd, nil + return nfd } -// Run nfdTopologyUpdater client. Returns if a fatal error is encountered, or, after +// Run nfdTopologyUpdater. Returns if a fatal error is encountered, or, after // one request if OneShot is set to 'true' in the updater args. func (w *nfdTopologyUpdater) Run() error { klog.Infof("Node Feature Discovery Topology Updater %s", version.Get()) - klog.Infof("NodeName: '%s'", nfdclient.NodeName()) + klog.Infof("NodeName: '%s'", w.nodeInfo.nodeName) podResClient, err := podres.GetPodResClient(w.resourcemonitorArgs.PodResourceSocketPath) if err != nil { return fmt.Errorf("failed to get PodResource Client: %w", err) } - var kubeApihelper apihelper.K8sHelpers if !w.args.NoPublish { kubeconfig, err := apihelper.GetKubeconfig(w.args.KubeConfigFile) if err != nil { return err } - kubeApihelper = apihelper.K8sHelpers{Kubeconfig: kubeconfig} + w.apihelper = apihelper.K8sHelpers{Kubeconfig: kubeconfig} } if err := w.configure(); err != nil { return fmt.Errorf("faild to configure Node Feature Discovery Topology Updater: %w", err) @@ -120,7 +114,7 @@ func (w *nfdTopologyUpdater) Run() error { var resScan resourcemonitor.ResourcesScanner - resScan, err = resourcemonitor.NewPodResourcesScanner(w.resourcemonitorArgs.Namespace, podResClient, kubeApihelper) + resScan, err = resourcemonitor.NewPodResourcesScanner(w.resourcemonitorArgs.Namespace, podResClient, w.apihelper) if err != nil { return fmt.Errorf("failed to initialize ResourceMonitor instance: %w", err) } @@ -131,7 +125,7 @@ func (w *nfdTopologyUpdater) Run() error { // zonesChannel := make(chan v1alpha1.ZoneList) var zones v1alpha1.ZoneList - excludeList := resourcemonitor.NewExcludeResourceList(w.config.ExcludeList, nfdclient.NodeName()) + excludeList := resourcemonitor.NewExcludeResourceList(w.config.ExcludeList, w.nodeInfo.nodeName) resAggr, err := resourcemonitor.NewResourcesAggregator(podResClient, excludeList) if err != nil { return fmt.Errorf("failed to obtain node resource information: %w", err) @@ -139,12 +133,6 @@ func (w *nfdTopologyUpdater) Run() error { klog.V(2).Infof("resAggr is: %v\n", resAggr) - // Create watcher for TLS certificates - w.certWatch, err = utils.CreateFsWatcher(time.Second, w.args.CaFile, w.args.CertFile, w.args.KeyFile) - if err != nil { - return err - } - crTrigger := time.NewTicker(w.resourcemonitorArgs.SleepInterval) for { select { @@ -158,49 +146,24 @@ func (w *nfdTopologyUpdater) Run() error { } zones = resAggr.Aggregate(podResources) utils.KlogDump(1, "After aggregating resources identified zones are", " ", zones) - if err = w.Update(zones); err != nil { - return err + if !w.args.NoPublish { + if err = w.updateNodeResourceTopology(zones); err != nil { + return err + } } if w.args.Oneshot { return nil } - case <-w.certWatch.Events: - klog.Infof("TLS certificate update, renewing connection to nfd-master") - w.Disconnect() - if err := w.Connect(); err != nil { - return err - } - case <-w.stop: klog.Infof("shutting down nfd-topology-updater") - w.certWatch.Close() return nil } } } -func (w *nfdTopologyUpdater) Update(zones v1alpha1.ZoneList) error { - // Connect to NFD master - err := w.Connect() - if err != nil { - return fmt.Errorf("failed to connect: %w", err) - } - defer w.Disconnect() - - if w.client == nil { - return nil - } - - err = advertiseNodeTopology(w.client, zones, w.nodeInfo.tmPolicy, nfdclient.NodeName()) - if err != nil { - return fmt.Errorf("failed to advertise node topology: %w", err) - } - return nil -} - // Stop NFD Topology Updater func (w *nfdTopologyUpdater) Stop() { select { @@ -209,59 +172,39 @@ func (w *nfdTopologyUpdater) Stop() { } } -// connect creates a client connection to the NFD master -func (w *nfdTopologyUpdater) Connect() error { - // Return a dummy connection in case of dry-run - if w.args.NoPublish { - return nil - } - - if err := w.NfdBaseClient.Connect(); err != nil { +func (w *nfdTopologyUpdater) updateNodeResourceTopology(zoneInfo v1alpha1.ZoneList) error { + cli, err := w.apihelper.GetTopologyClient() + if err != nil { return err } - w.client = pb.NewNodeTopologyClient(w.ClientConn()) - - return nil -} -// disconnect closes the connection to NFD master -func (w *nfdTopologyUpdater) Disconnect() { - w.NfdBaseClient.Disconnect() - w.client = nil -} - -// advertiseNodeTopology advertises the topology CR to a Kubernetes node -// via the NFD server. -func advertiseNodeTopology(client pb.NodeTopologyClient, zoneInfo v1alpha1.ZoneList, tmPolicy string, nodeName string) error { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - zones := make([]*v1alpha1.Zone, len(zoneInfo)) - // TODO: Avoid copying of data to allow returning the zone info - // directly in a compatible data type (i.e. []*v1alpha1.Zone). - for i, zone := range zoneInfo { - zones[i] = &v1alpha1.Zone{ - Name: zone.Name, - Type: zone.Type, - Parent: zone.Parent, - Resources: zone.Resources, - Costs: zone.Costs, + nrt, err := cli.TopologyV1alpha1().NodeResourceTopologies().Get(context.TODO(), w.nodeInfo.nodeName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + nrtNew := v1alpha1.NodeResourceTopology{ + ObjectMeta: metav1.ObjectMeta{ + Name: w.nodeInfo.nodeName, + }, + Zones: zoneInfo, + TopologyPolicies: []string{w.nodeInfo.tmPolicy}, } - } - topologyReq := &pb.NodeTopologyRequest{ - Zones: zones, - NfdVersion: version.Get(), - NodeName: nodeName, - TopologyPolicies: []string{tmPolicy}, + _, err := cli.TopologyV1alpha1().NodeResourceTopologies().Create(context.TODO(), &nrtNew, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create NodeResourceTopology: %w", err) + } + return nil + } else if err != nil { + return err } - utils.KlogDump(1, "Sending NodeTopologyRequest to nfd-master:", " ", topologyReq) + nrtMutated := nrt.DeepCopy() + nrtMutated.Zones = zoneInfo - _, err := client.UpdateNodeTopology(ctx, topologyReq) + nrtUpdated, err := cli.TopologyV1alpha1().NodeResourceTopologies().Update(context.TODO(), nrtMutated, metav1.UpdateOptions{}) if err != nil { - return err + return fmt.Errorf("failed to update NodeResourceTopology: %w", err) } - + utils.KlogDump(4, "CR instance updated resTopo:", " ", nrtUpdated) return nil } diff --git a/pkg/nfd-client/topology-updater/nfd-topology-updater_test.go b/pkg/nfd-client/topology-updater/nfd-topology-updater_test.go deleted file mode 100644 index 9f8434b998..0000000000 --- a/pkg/nfd-client/topology-updater/nfd-topology-updater_test.go +++ /dev/null @@ -1,169 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package topologyupdater_test - -import ( - "fmt" - "os" - "testing" - "time" - - v1alpha1 "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/apis/topology/v1alpha1" - . "github.com/smartystreets/goconvey/convey" - "k8s.io/apimachinery/pkg/api/resource" - nfdclient "sigs.k8s.io/node-feature-discovery/pkg/nfd-client" - u "sigs.k8s.io/node-feature-discovery/pkg/nfd-client/topology-updater" - nfdmaster "sigs.k8s.io/node-feature-discovery/pkg/nfd-master" - "sigs.k8s.io/node-feature-discovery/pkg/resourcemonitor" - "sigs.k8s.io/node-feature-discovery/test/data" -) - -type testContext struct { - master nfdmaster.NfdMaster - errs chan error -} - -func setupTest(args *nfdmaster.Args) testContext { - // Fixed port and no-publish, for convenience - args.NoPublish = true - args.Port = 8192 - m, err := nfdmaster.NewNfdMaster(args) - if err != nil { - fmt.Printf("Test setup failed: %v\n", err) - os.Exit(1) - } - ctx := testContext{master: m, errs: make(chan error)} - - // Run nfd-master instance, intended to be used as the server counterpart - go func() { - ctx.errs <- ctx.master.Run() - close(ctx.errs) - }() - ready := ctx.master.WaitForReady(5 * time.Second) - if !ready { - fmt.Println("Test setup failed: timeout while waiting for nfd-master") - os.Exit(1) - } - - return ctx -} - -func teardownTest(ctx testContext) { - ctx.master.Stop() - for e := range ctx.errs { - if e != nil { - fmt.Printf("Error in test context: %v\n", e) - os.Exit(1) - } - } -} - -func TestNewTopologyUpdater(t *testing.T) { - Convey("When initializing new NfdTopologyUpdater instance", t, func() { - Convey("When one of -cert-file, -key-file or -ca-file is missing", func() { - tmPolicy := "fake-topology-manager-policy" - _, err := u.NewTopologyUpdater(u.Args{Args: nfdclient.Args{CertFile: "crt", KeyFile: "key"}}, resourcemonitor.Args{}, tmPolicy) - _, err2 := u.NewTopologyUpdater(u.Args{Args: nfdclient.Args{KeyFile: "key", CaFile: "ca"}}, resourcemonitor.Args{}, tmPolicy) - _, err3 := u.NewTopologyUpdater(u.Args{Args: nfdclient.Args{CertFile: "crt", CaFile: "ca"}}, resourcemonitor.Args{}, tmPolicy) - Convey("An error should be returned", func() { - So(err, ShouldNotBeNil) - So(err2, ShouldNotBeNil) - So(err3, ShouldNotBeNil) - }) - }) - }) -} - -func TestUpdate(t *testing.T) { - ctx := setupTest(&nfdmaster.Args{}) - resourceInfo := v1alpha1.ResourceInfoList{ - v1alpha1.ResourceInfo{ - Name: "cpu", - Available: resource.MustParse("2"), - Allocatable: resource.MustParse("4"), - Capacity: resource.MustParse("4"), - }, - } - zones := v1alpha1.ZoneList{ - v1alpha1.Zone{ - Name: "node-0", - Type: "Node", - Resources: resourceInfo, - }, - } - defer teardownTest(ctx) - Convey("When running nfd-topology-updater against nfd-master", t, func() { - Convey("When running as a Oneshot job with Zones", func() { - args := u.Args{ - Oneshot: true, - Args: nfdclient.Args{ - Server: "localhost:8192"}, - } - updater, _ := u.NewTopologyUpdater(args, resourcemonitor.Args{}, "fake-topology-manager-policy") - err := updater.Update(zones) - Convey("No error should be returned", func() { - So(err, ShouldBeNil) - }) - }) - }) -} - -func TestRunTls(t *testing.T) { - masterArgs := &nfdmaster.Args{ - CaFile: data.FilePath("ca.crt"), - CertFile: data.FilePath("nfd-test-master.crt"), - KeyFile: data.FilePath("nfd-test-master.key"), - VerifyNodeName: false, - } - ctx := setupTest(masterArgs) - defer teardownTest(ctx) - Convey("When running nfd-worker against nfd-master with mutual TLS auth enabled", t, func() { - Convey("When publishing CRs obtained from Zones", func() { - resourceInfo := v1alpha1.ResourceInfoList{ - v1alpha1.ResourceInfo{ - Name: "cpu", - Available: resource.MustParse("2"), - Allocatable: resource.MustParse("4"), - Capacity: resource.MustParse("4"), - }, - } - zones := v1alpha1.ZoneList{ - v1alpha1.Zone{ - Name: "node-0", - Type: "Node", - Resources: resourceInfo, - }, - } - updaterArgs := u.Args{ - Args: nfdclient.Args{ - CaFile: data.FilePath("ca.crt"), - CertFile: data.FilePath("nfd-test-topology-updater.crt"), - KeyFile: data.FilePath("nfd-test-topology-updater.key"), - Server: "localhost:8192", - ServerNameOverride: "nfd-test-master", - }, - Oneshot: true, - } - - updater, _ := u.NewTopologyUpdater(updaterArgs, resourcemonitor.Args{}, "fake-topology-manager-policy") - err := updater.Update(zones) - Convey("No error should be returned", func() { - So(err, ShouldBeNil) - }) - }) - }) -} diff --git a/pkg/nfd-master/nfd-master.go b/pkg/nfd-master/nfd-master.go index d46f16bd32..29c88bbd0b 100644 --- a/pkg/nfd-master/nfd-master.go +++ b/pkg/nfd-master/nfd-master.go @@ -29,7 +29,6 @@ import ( "strings" "time" - "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/apis/topology/v1alpha1" "golang.org/x/net/context" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -37,8 +36,6 @@ import ( "google.golang.org/grpc/health/grpc_health_v1" "google.golang.org/grpc/peer" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" label "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" @@ -49,7 +46,6 @@ import ( "sigs.k8s.io/node-feature-discovery/pkg/apihelper" nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/pkg/apis/nfd/v1alpha1" pb "sigs.k8s.io/node-feature-discovery/pkg/labeler" - topologypb "sigs.k8s.io/node-feature-discovery/pkg/topologyupdater" "sigs.k8s.io/node-feature-discovery/pkg/utils" "sigs.k8s.io/node-feature-discovery/pkg/version" ) @@ -220,7 +216,6 @@ func (m *nfdMaster) runGrpcServer(errChan chan<- error) { m.server = grpc.NewServer(serverOpts...) pb.RegisterLabelerServer(m.server, m) grpc_health_v1.RegisterHealthServer(m.server, health.NewServer()) - topologypb.RegisterNodeTopologyServer(m.server, m) klog.Infof("gRPC server serving on port: %d", m.args.Port) // Run gRPC server @@ -574,27 +569,6 @@ func authorizeClient(c context.Context, checkNodeName bool, nodeName string) err return nil } -func (m *nfdMaster) UpdateNodeTopology(c context.Context, r *topologypb.NodeTopologyRequest) (*topologypb.NodeTopologyResponse, error) { - err := authorizeClient(c, m.args.VerifyNodeName, r.NodeName) - if err != nil { - return &topologypb.NodeTopologyResponse{}, err - } - if klog.V(1).Enabled() { - klog.Infof("REQUEST Node: %s NFD-version: %s Topology Policy: %s", r.NodeName, r.NfdVersion, r.TopologyPolicies) - utils.KlogDump(1, "Zones received:", " ", r.Zones) - } else { - klog.Infof("received CR updation request for node %q", r.NodeName) - } - if !m.args.NoPublish { - err := m.updateCR(r.NodeName, r.TopologyPolicies, r.Zones) - if err != nil { - klog.Errorf("failed to advertise NodeResourceTopology: %v", err) - return &topologypb.NodeTopologyResponse{}, err - } - } - return &topologypb.NodeTopologyResponse{}, nil -} - func (m *nfdMaster) processNodeFeatureRule(r *pb.SetLabelsRequest) (map[string]string, []corev1.Taint) { if m.nfdController == nil { return nil, nil @@ -806,60 +780,6 @@ func stringToNsNames(cslist, ns string) []string { return names } -func modifyCR(topoUpdaterZones []*v1alpha1.Zone) []v1alpha1.Zone { - zones := make([]v1alpha1.Zone, len(topoUpdaterZones)) - // TODO: Avoid copying of data to allow returning the zone info - // directly in a compatible data type (i.e. []*v1alpha1.Zone). - for i, zone := range topoUpdaterZones { - zones[i] = v1alpha1.Zone{ - Name: zone.Name, - Type: zone.Type, - Parent: zone.Parent, - Costs: zone.Costs, - Resources: zone.Resources, - } - } - return zones -} - -func (m *nfdMaster) updateCR(hostname string, tmpolicy []string, topoUpdaterZones []*v1alpha1.Zone) error { - cli, err := m.apihelper.GetTopologyClient() - if err != nil { - return err - } - - zones := modifyCR(topoUpdaterZones) - - nrt, err := cli.TopologyV1alpha1().NodeResourceTopologies().Get(context.TODO(), hostname, metav1.GetOptions{}) - if errors.IsNotFound(err) { - nrtNew := v1alpha1.NodeResourceTopology{ - ObjectMeta: metav1.ObjectMeta{ - Name: hostname, - }, - Zones: zones, - TopologyPolicies: tmpolicy, - } - - _, err := cli.TopologyV1alpha1().NodeResourceTopologies().Create(context.TODO(), &nrtNew, metav1.CreateOptions{}) - if err != nil { - return fmt.Errorf("failed to create v1alpha1.NodeResourceTopology!:%w", err) - } - return nil - } else if err != nil { - return err - } - - nrtMutated := nrt.DeepCopy() - nrtMutated.Zones = zones - - nrtUpdated, err := cli.TopologyV1alpha1().NodeResourceTopologies().Update(context.TODO(), nrtMutated, metav1.UpdateOptions{}) - if err != nil { - return fmt.Errorf("failed to update v1alpha1.NodeResourceTopology!:%w", err) - } - utils.KlogDump(2, "CR instance updated resTopo:", " ", nrtUpdated) - return nil -} - func (m *nfdMaster) instanceAnnotation(name string) string { if m.args.Instance == "" { return name diff --git a/pkg/topologyupdater/doc.go b/pkg/topologyupdater/doc.go deleted file mode 100644 index b95359c68b..0000000000 --- a/pkg/topologyupdater/doc.go +++ /dev/null @@ -1,20 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package topologyupdater - -//go:generate protoc --go_opt=paths=source_relative --go_out=plugins=grpc:. -I . -I ../../vendor/ topology-updater.proto -//go:generate mockery --name=NodeTopologyClient --inpackage diff --git a/pkg/topologyupdater/mock_NodeTopologyClient.go b/pkg/topologyupdater/mock_NodeTopologyClient.go deleted file mode 100644 index 3ee715c274..0000000000 --- a/pkg/topologyupdater/mock_NodeTopologyClient.go +++ /dev/null @@ -1,61 +0,0 @@ -// Code generated by mockery v2.13.0. DO NOT EDIT. - -package topologyupdater - -import ( - context "context" - - grpc "google.golang.org/grpc" - - mock "github.com/stretchr/testify/mock" -) - -// MockNodeTopologyClient is an autogenerated mock type for the NodeTopologyClient type -type MockNodeTopologyClient struct { - mock.Mock -} - -// UpdateNodeTopology provides a mock function with given fields: ctx, in, opts -func (_m *MockNodeTopologyClient) UpdateNodeTopology(ctx context.Context, in *NodeTopologyRequest, opts ...grpc.CallOption) (*NodeTopologyResponse, error) { - _va := make([]interface{}, len(opts)) - for _i := range opts { - _va[_i] = opts[_i] - } - var _ca []interface{} - _ca = append(_ca, ctx, in) - _ca = append(_ca, _va...) - ret := _m.Called(_ca...) - - var r0 *NodeTopologyResponse - if rf, ok := ret.Get(0).(func(context.Context, *NodeTopologyRequest, ...grpc.CallOption) *NodeTopologyResponse); ok { - r0 = rf(ctx, in, opts...) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*NodeTopologyResponse) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, *NodeTopologyRequest, ...grpc.CallOption) error); ok { - r1 = rf(ctx, in, opts...) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -type NewMockNodeTopologyClientT interface { - mock.TestingT - Cleanup(func()) -} - -// NewMockNodeTopologyClient creates a new instance of MockNodeTopologyClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewMockNodeTopologyClient(t NewMockNodeTopologyClientT) *MockNodeTopologyClient { - mock := &MockNodeTopologyClient{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/pkg/topologyupdater/topology-updater.pb.go b/pkg/topologyupdater/topology-updater.pb.go deleted file mode 100644 index 41d45d54f0..0000000000 --- a/pkg/topologyupdater/topology-updater.pb.go +++ /dev/null @@ -1,353 +0,0 @@ -// -//Copyright 2021 The Kubernetes Authors. -// -//Licensed under the Apache License, Version 2.0 (the "License"); -//you may not use this file except in compliance with the License. -//You may obtain a copy of the License at -// -//http://www.apache.org/licenses/LICENSE-2.0 -// -//Unless required by applicable law or agreed to in writing, software -//distributed under the License is distributed on an "AS IS" BASIS, -//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -//See the License for the specific language governing permissions and -//limitations under the License. - -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.23.0 -// protoc v3.17.3 -// source: topology-updater.proto - -package topologyupdater - -import ( - context "context" - proto "github.com/golang/protobuf/proto" - v1alpha1 "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/apis/topology/v1alpha1" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - -type NodeTopologyRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - NfdVersion string `protobuf:"bytes,1,opt,name=nfd_version,json=nfdVersion,proto3" json:"nfd_version,omitempty"` - NodeName string `protobuf:"bytes,2,opt,name=node_name,json=nodeName,proto3" json:"node_name,omitempty"` - TopologyPolicies []string `protobuf:"bytes,3,rep,name=topology_policies,json=topologyPolicies,proto3" json:"topology_policies,omitempty"` - Zones []*v1alpha1.Zone `protobuf:"bytes,4,rep,name=zones,proto3" json:"zones,omitempty"` -} - -func (x *NodeTopologyRequest) Reset() { - *x = NodeTopologyRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_topology_updater_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *NodeTopologyRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*NodeTopologyRequest) ProtoMessage() {} - -func (x *NodeTopologyRequest) ProtoReflect() protoreflect.Message { - mi := &file_topology_updater_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use NodeTopologyRequest.ProtoReflect.Descriptor instead. -func (*NodeTopologyRequest) Descriptor() ([]byte, []int) { - return file_topology_updater_proto_rawDescGZIP(), []int{0} -} - -func (x *NodeTopologyRequest) GetNfdVersion() string { - if x != nil { - return x.NfdVersion - } - return "" -} - -func (x *NodeTopologyRequest) GetNodeName() string { - if x != nil { - return x.NodeName - } - return "" -} - -func (x *NodeTopologyRequest) GetTopologyPolicies() []string { - if x != nil { - return x.TopologyPolicies - } - return nil -} - -func (x *NodeTopologyRequest) GetZones() []*v1alpha1.Zone { - if x != nil { - return x.Zones - } - return nil -} - -type NodeTopologyResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *NodeTopologyResponse) Reset() { - *x = NodeTopologyResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_topology_updater_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *NodeTopologyResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*NodeTopologyResponse) ProtoMessage() {} - -func (x *NodeTopologyResponse) ProtoReflect() protoreflect.Message { - mi := &file_topology_updater_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use NodeTopologyResponse.ProtoReflect.Descriptor instead. -func (*NodeTopologyResponse) Descriptor() ([]byte, []int) { - return file_topology_updater_proto_rawDescGZIP(), []int{1} -} - -var File_topology_updater_proto protoreflect.FileDescriptor - -var file_topology_updater_proto_rawDesc = []byte{ - 0x0a, 0x16, 0x74, 0x6f, 0x70, 0x6f, 0x6c, 0x6f, 0x67, 0x79, 0x2d, 0x75, 0x70, 0x64, 0x61, 0x74, - 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0f, 0x74, 0x6f, 0x70, 0x6f, 0x6c, 0x6f, - 0x67, 0x79, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x72, 0x1a, 0x66, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6b, 0x38, 0x73, 0x74, 0x6f, 0x70, 0x6f, 0x6c, 0x6f, 0x67, - 0x79, 0x61, 0x77, 0x61, 0x72, 0x65, 0x73, 0x63, 0x68, 0x65, 0x64, 0x77, 0x67, 0x2f, 0x6e, 0x6f, - 0x64, 0x65, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x74, 0x6f, 0x70, 0x6f, 0x6c, 0x6f, - 0x67, 0x79, 0x2d, 0x61, 0x70, 0x69, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, - 0x74, 0x6f, 0x70, 0x6f, 0x6c, 0x6f, 0x67, 0x79, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, - 0x31, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x22, 0xa6, 0x01, 0x0a, 0x13, 0x4e, 0x6f, 0x64, 0x65, 0x54, 0x6f, 0x70, 0x6f, 0x6c, 0x6f, - 0x67, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x66, 0x64, - 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, - 0x6e, 0x66, 0x64, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x6e, 0x6f, - 0x64, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6e, - 0x6f, 0x64, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2b, 0x0a, 0x11, 0x74, 0x6f, 0x70, 0x6f, 0x6c, - 0x6f, 0x67, 0x79, 0x5f, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x10, 0x74, 0x6f, 0x70, 0x6f, 0x6c, 0x6f, 0x67, 0x79, 0x50, 0x6f, 0x6c, 0x69, - 0x63, 0x69, 0x65, 0x73, 0x12, 0x24, 0x0a, 0x05, 0x7a, 0x6f, 0x6e, 0x65, 0x73, 0x18, 0x04, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x5a, - 0x6f, 0x6e, 0x65, 0x52, 0x05, 0x7a, 0x6f, 0x6e, 0x65, 0x73, 0x22, 0x16, 0x0a, 0x14, 0x4e, 0x6f, - 0x64, 0x65, 0x54, 0x6f, 0x70, 0x6f, 0x6c, 0x6f, 0x67, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x32, 0x71, 0x0a, 0x0c, 0x4e, 0x6f, 0x64, 0x65, 0x54, 0x6f, 0x70, 0x6f, 0x6c, 0x6f, - 0x67, 0x79, 0x12, 0x61, 0x0a, 0x12, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, - 0x54, 0x6f, 0x70, 0x6f, 0x6c, 0x6f, 0x67, 0x79, 0x12, 0x24, 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x6c, - 0x6f, 0x67, 0x79, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x72, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x54, - 0x6f, 0x70, 0x6f, 0x6c, 0x6f, 0x67, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, - 0x2e, 0x74, 0x6f, 0x70, 0x6f, 0x6c, 0x6f, 0x67, 0x79, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x72, - 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x54, 0x6f, 0x70, 0x6f, 0x6c, 0x6f, 0x67, 0x79, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x38, 0x5a, 0x36, 0x73, 0x69, 0x67, 0x73, 0x2e, 0x6b, 0x38, - 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x6e, 0x6f, 0x64, 0x65, 0x2d, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x2d, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x2f, 0x70, 0x6b, 0x67, 0x2f, - 0x74, 0x6f, 0x70, 0x6f, 0x6c, 0x6f, 0x67, 0x79, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x72, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_topology_updater_proto_rawDescOnce sync.Once - file_topology_updater_proto_rawDescData = file_topology_updater_proto_rawDesc -) - -func file_topology_updater_proto_rawDescGZIP() []byte { - file_topology_updater_proto_rawDescOnce.Do(func() { - file_topology_updater_proto_rawDescData = protoimpl.X.CompressGZIP(file_topology_updater_proto_rawDescData) - }) - return file_topology_updater_proto_rawDescData -} - -var file_topology_updater_proto_msgTypes = make([]protoimpl.MessageInfo, 2) -var file_topology_updater_proto_goTypes = []interface{}{ - (*NodeTopologyRequest)(nil), // 0: topologyupdater.NodeTopologyRequest - (*NodeTopologyResponse)(nil), // 1: topologyupdater.NodeTopologyResponse - (*v1alpha1.Zone)(nil), // 2: v1alpha1.Zone -} -var file_topology_updater_proto_depIdxs = []int32{ - 2, // 0: topologyupdater.NodeTopologyRequest.zones:type_name -> v1alpha1.Zone - 0, // 1: topologyupdater.NodeTopology.UpdateNodeTopology:input_type -> topologyupdater.NodeTopologyRequest - 1, // 2: topologyupdater.NodeTopology.UpdateNodeTopology:output_type -> topologyupdater.NodeTopologyResponse - 2, // [2:3] is the sub-list for method output_type - 1, // [1:2] is the sub-list for method input_type - 1, // [1:1] is the sub-list for extension type_name - 1, // [1:1] is the sub-list for extension extendee - 0, // [0:1] is the sub-list for field type_name -} - -func init() { file_topology_updater_proto_init() } -func file_topology_updater_proto_init() { - if File_topology_updater_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_topology_updater_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeTopologyRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_topology_updater_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeTopologyResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_topology_updater_proto_rawDesc, - NumEnums: 0, - NumMessages: 2, - NumExtensions: 0, - NumServices: 1, - }, - GoTypes: file_topology_updater_proto_goTypes, - DependencyIndexes: file_topology_updater_proto_depIdxs, - MessageInfos: file_topology_updater_proto_msgTypes, - }.Build() - File_topology_updater_proto = out.File - file_topology_updater_proto_rawDesc = nil - file_topology_updater_proto_goTypes = nil - file_topology_updater_proto_depIdxs = nil -} - -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConnInterface - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion6 - -// NodeTopologyClient is the client API for NodeTopology service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type NodeTopologyClient interface { - UpdateNodeTopology(ctx context.Context, in *NodeTopologyRequest, opts ...grpc.CallOption) (*NodeTopologyResponse, error) -} - -type nodeTopologyClient struct { - cc grpc.ClientConnInterface -} - -func NewNodeTopologyClient(cc grpc.ClientConnInterface) NodeTopologyClient { - return &nodeTopologyClient{cc} -} - -func (c *nodeTopologyClient) UpdateNodeTopology(ctx context.Context, in *NodeTopologyRequest, opts ...grpc.CallOption) (*NodeTopologyResponse, error) { - out := new(NodeTopologyResponse) - err := c.cc.Invoke(ctx, "/topologyupdater.NodeTopology/UpdateNodeTopology", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// NodeTopologyServer is the server API for NodeTopology service. -type NodeTopologyServer interface { - UpdateNodeTopology(context.Context, *NodeTopologyRequest) (*NodeTopologyResponse, error) -} - -// UnimplementedNodeTopologyServer can be embedded to have forward compatible implementations. -type UnimplementedNodeTopologyServer struct { -} - -func (*UnimplementedNodeTopologyServer) UpdateNodeTopology(context.Context, *NodeTopologyRequest) (*NodeTopologyResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method UpdateNodeTopology not implemented") -} - -func RegisterNodeTopologyServer(s *grpc.Server, srv NodeTopologyServer) { - s.RegisterService(&_NodeTopology_serviceDesc, srv) -} - -func _NodeTopology_UpdateNodeTopology_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(NodeTopologyRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(NodeTopologyServer).UpdateNodeTopology(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/topologyupdater.NodeTopology/UpdateNodeTopology", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(NodeTopologyServer).UpdateNodeTopology(ctx, req.(*NodeTopologyRequest)) - } - return interceptor(ctx, in, info, handler) -} - -var _NodeTopology_serviceDesc = grpc.ServiceDesc{ - ServiceName: "topologyupdater.NodeTopology", - HandlerType: (*NodeTopologyServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "UpdateNodeTopology", - Handler: _NodeTopology_UpdateNodeTopology_Handler, - }, - }, - Streams: []grpc.StreamDesc{}, - Metadata: "topology-updater.proto", -} diff --git a/pkg/topologyupdater/topology-updater.proto b/pkg/topologyupdater/topology-updater.proto deleted file mode 100644 index 2cbcc44975..0000000000 --- a/pkg/topologyupdater/topology-updater.proto +++ /dev/null @@ -1,35 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -syntax = "proto3"; - -option go_package = "sigs.k8s.io/node-feature-discovery/pkg/topologyupdater"; -import "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/apis/topology/v1alpha1/generated.proto"; - -package topologyupdater; - -service NodeTopology{ - rpc UpdateNodeTopology(NodeTopologyRequest) returns (NodeTopologyResponse); -} - -message NodeTopologyRequest { - string nfd_version = 1; - string node_name = 2; - repeated string topology_policies = 3; - repeated v1alpha1.Zone zones = 4; -} - -message NodeTopologyResponse { -} diff --git a/test/e2e/topology_updater.go b/test/e2e/topology_updater.go index 05e2b65f26..fb9ec48220 100644 --- a/test/e2e/topology_updater.go +++ b/test/e2e/topology_updater.go @@ -19,9 +19,10 @@ package e2e import ( "context" "fmt" - "k8s.io/apimachinery/pkg/api/resource" "time" + "k8s.io/apimachinery/pkg/api/resource" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -36,7 +37,6 @@ import ( kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework/kubelet" - e2enetwork "k8s.io/kubernetes/test/e2e/framework/network" admissionapi "k8s.io/pod-security-admission/api" testutils "sigs.k8s.io/node-feature-discovery/test/e2e/utils" @@ -72,18 +72,9 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { By("Creating the node resource topologies CRD") Expect(testutils.CreateNodeResourceTopologies(extClient)).ToNot(BeNil()) + By("Configuring RBAC") Expect(testutils.ConfigureRBAC(f.ClientSet, f.Namespace.Name)).NotTo(HaveOccurred()) - imageOpt := testpod.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)) - f.PodClient().CreateSync(testpod.NFDMaster(imageOpt)) - - // Create nfd-master service - masterService, err := testutils.CreateService(f.ClientSet, f.Namespace.Name) - Expect(err).NotTo(HaveOccurred()) - - By("Waiting for the nfd-master service to be up") - Expect(e2enetwork.WaitForService(f.ClientSet, f.Namespace.Name, masterService.Name, true, time.Second, 10*time.Second)).NotTo(HaveOccurred()) - By("Creating nfd-topology-updater daemonset") topologyUpdaterDaemonSet, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), topologyUpdaterDaemonSet, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -114,7 +105,7 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { } }) - Context("with single nfd-master pod", func() { + Context("with topology-updater daemonset running", func() { BeforeEach(func() { cfg, err := testutils.GetConfig() Expect(err).ToNot(HaveOccurred()) diff --git a/test/e2e/utils/pod/pod.go b/test/e2e/utils/pod/pod.go index 82485569cc..0e92c0df76 100644 --- a/test/e2e/utils/pod/pod.go +++ b/test/e2e/utils/pod/pod.go @@ -367,8 +367,7 @@ func NFDTopologyUpdaterSpec(kc utils.KubeletConfig, opts ...SpecOption) *corev1. "--kubelet-config-uri=file:///podresources/config.yaml", "--podresources-socket=unix:///podresources/kubelet.sock", "--sleep-interval=3s", - "--watch-namespace=rte", - "--server=nfd-master-e2e:8080"}, + "--watch-namespace=rte"}, Env: []corev1.EnvVar{ { Name: "NODE_NAME", diff --git a/test/e2e/utils/rbac.go b/test/e2e/utils/rbac.go index 14add8009a..9d851576b1 100644 --- a/test/e2e/utils/rbac.go +++ b/test/e2e/utils/rbac.go @@ -133,15 +133,6 @@ func createClusterRoleMaster(cs clientset.Interface) (*rbacv1.ClusterRole, error Resources: []string{"nodefeaturerules"}, Verbs: []string{"get", "list", "watch"}, }, - { - APIGroups: []string{"topology.node.k8s.io"}, - Resources: []string{"noderesourcetopologies"}, - Verbs: []string{ - "create", - "get", - "update", - }, - }, }, } if *openShift { @@ -172,6 +163,15 @@ func createClusterRoleTopologyUpdater(cs clientset.Interface) (*rbacv1.ClusterRo Resources: []string{"pods"}, Verbs: []string{"get", "list", "watch"}, }, + { + APIGroups: []string{"topology.node.k8s.io"}, + Resources: []string{"noderesourcetopologies"}, + Verbs: []string{ + "create", + "get", + "update", + }, + }, }, } if *openShift { From 409312e1111b663ea137f0509171e8d952bea0bd Mon Sep 17 00:00:00 2001 From: Feruzjon Muyassarov Date: Mon, 12 Dec 2022 16:29:46 +0200 Subject: [PATCH 39/87] Bump go.mod k8s.io to 1.26 Signed-off-by: Feruzjon Muyassarov --- go.mod | 186 ++++++++------- go.sum | 350 ++++++++++++++++------------- test/e2e/node_feature_discovery.go | 2 +- test/e2e/utils/config.go | 2 +- 4 files changed, 283 insertions(+), 257 deletions(-) diff --git a/go.mod b/go.mod index facd889027..bc723481c9 100644 --- a/go.mod +++ b/go.mod @@ -3,34 +3,34 @@ module sigs.k8s.io/node-feature-discovery go 1.19 require ( - github.com/fsnotify/fsnotify v1.4.9 + github.com/fsnotify/fsnotify v1.6.0 github.com/ghodss/yaml v1.0.0 github.com/gogo/protobuf v1.3.2 github.com/golang/protobuf v1.5.2 - github.com/google/go-cmp v0.5.8 + github.com/google/go-cmp v0.5.9 github.com/jaypipes/ghw v0.8.1-0.20210827132705-c7224150a17e github.com/k8stopologyawareschedwg/noderesourcetopology-api v0.0.12 github.com/klauspost/cpuid/v2 v2.2.2 - github.com/onsi/ginkgo/v2 v2.3.1 - github.com/onsi/gomega v1.22.1 + github.com/onsi/ginkgo/v2 v2.4.0 + github.com/onsi/gomega v1.23.0 github.com/smartystreets/assertions v1.2.0 github.com/smartystreets/goconvey v1.6.4 github.com/stretchr/testify v1.8.0 github.com/vektra/errors v0.0.0-20140903201135-c64d83aba85a - golang.org/x/net v0.0.0-20220722155237-a158d28d115b - google.golang.org/grpc v1.47.0 - google.golang.org/protobuf v1.28.0 - k8s.io/api v0.25.3 - k8s.io/apiextensions-apiserver v0.25.3 - k8s.io/apimachinery v0.25.3 - k8s.io/client-go v0.25.3 - k8s.io/klog/v2 v2.70.1 - k8s.io/kubectl v0.25.3 - k8s.io/kubelet v0.25.3 - k8s.io/kubernetes v1.25.3 + golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10 + google.golang.org/grpc v1.49.0 + google.golang.org/protobuf v1.28.1 + k8s.io/api v0.26.0 + k8s.io/apiextensions-apiserver v0.26.0 + k8s.io/apimachinery v0.26.0 + k8s.io/client-go v0.26.0 + k8s.io/klog/v2 v2.80.1 + k8s.io/kubectl v0.26.0 + k8s.io/kubelet v0.26.0 + k8s.io/kubernetes v1.26.0 k8s.io/pod-security-admission v0.0.0 - k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed - sigs.k8s.io/yaml v1.2.0 + k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 + sigs.k8s.io/yaml v1.3.0 ) require ( @@ -50,72 +50,68 @@ require ( github.com/Microsoft/go-winio v0.4.17 // indirect github.com/Microsoft/hcsshim v0.8.22 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect - github.com/PuerkitoBio/purell v1.1.1 // indirect - github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect + github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e // indirect - github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect - github.com/aws/aws-sdk-go v1.38.49 // indirect + github.com/aws/aws-sdk-go v1.44.116 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/checkpoint-restore/go-criu/v5 v5.3.0 // indirect github.com/cilium/ebpf v0.7.0 // indirect - github.com/container-storage-interface/spec v1.6.0 // indirect + github.com/container-storage-interface/spec v1.7.0 // indirect github.com/containerd/cgroups v1.0.1 // indirect github.com/containerd/console v1.0.3 // indirect - github.com/containerd/ttrpc v1.0.2 // indirect + github.com/containerd/ttrpc v1.1.0 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/cyphar/filepath-securejoin v0.2.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/distribution v2.8.1+incompatible // indirect - github.com/docker/go-units v0.4.0 // indirect - github.com/emicklei/go-restful/v3 v3.8.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/euank/go-kmsg-parser v2.0.0+incompatible // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect - github.com/felixge/httpsnoop v1.0.1 // indirect - github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect + github.com/felixge/httpsnoop v1.0.3 // indirect github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.4 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.20.0 // indirect github.com/go-openapi/swag v0.19.14 // indirect - github.com/go-ozzo/ozzo-validation v3.5.0+incompatible // indirect github.com/godbus/dbus/v5 v5.0.6 // indirect github.com/gofrs/uuid v4.0.0+incompatible // indirect github.com/golang-jwt/jwt/v4 v4.2.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/mock v1.6.0 // indirect - github.com/google/cadvisor v0.45.0 // indirect + github.com/google/cadvisor v0.46.0 // indirect + github.com/google/cel-go v0.12.5 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/gofuzz v1.1.0 // indirect github.com/google/uuid v1.1.2 // indirect github.com/googleapis/gax-go/v2 v2.1.1 // indirect - github.com/gophercloud/gophercloud v0.1.0 // indirect github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect - github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect - github.com/heketi/heketi v10.3.0+incompatible // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect github.com/imdario/mergo v0.3.6 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/jaypipes/pcidb v0.6.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect - github.com/karrick/godirwalk v1.16.1 // indirect + github.com/karrick/godirwalk v1.17.0 // indirect github.com/libopenstorage/openstorage v1.0.0 // indirect github.com/lithammer/dedent v1.1.0 // indirect github.com/mailru/easyjson v0.7.6 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989 // indirect github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/moby/ipvs v1.0.1 // indirect github.com/moby/spdystream v0.2.0 // indirect - github.com/moby/sys/mountinfo v0.6.0 // indirect + github.com/moby/sys/mountinfo v0.6.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mohae/deepcopy v0.0.0-20170603005431-491d3605edfb // indirect @@ -123,50 +119,50 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/runc v1.1.3 // indirect + github.com/opencontainers/runc v1.1.4 // indirect github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect github.com/opencontainers/selinux v1.10.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.12.1 // indirect - github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.32.1 // indirect - github.com/prometheus/procfs v0.7.3 // indirect + github.com/prometheus/client_golang v1.14.0 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.37.0 // indirect + github.com/prometheus/procfs v0.8.0 // indirect github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021 // indirect github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 // indirect github.com/sirupsen/logrus v1.8.1 // indirect - github.com/spf13/cobra v1.4.0 // indirect + github.com/spf13/cobra v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stoewer/go-strcase v1.2.0 // indirect github.com/stretchr/objx v0.4.0 // indirect github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect github.com/vishvananda/netlink v1.1.0 // indirect github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect github.com/vmware/govmomi v0.20.3 // indirect - go.etcd.io/etcd/api/v3 v3.5.4 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect - go.etcd.io/etcd/client/v3 v3.5.4 // indirect + go.etcd.io/etcd/api/v3 v3.5.5 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect + go.etcd.io/etcd/client/v3 v3.5.5 // indirect go.opencensus.io v0.23.0 // indirect - go.opentelemetry.io/contrib v0.20.0 // indirect - go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.20.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0 // indirect - go.opentelemetry.io/otel v0.20.0 // indirect - go.opentelemetry.io/otel/exporters/otlp v0.20.0 // indirect - go.opentelemetry.io/otel/metric v0.20.0 // indirect - go.opentelemetry.io/otel/sdk v0.20.0 // indirect - go.opentelemetry.io/otel/sdk/export/metric v0.20.0 // indirect - go.opentelemetry.io/otel/sdk/metric v0.20.0 // indirect - go.opentelemetry.io/otel/trace v0.20.0 // indirect - go.opentelemetry.io/proto/otlp v0.7.0 // indirect + go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.35.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0 // indirect + go.opentelemetry.io/otel v1.10.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 // indirect + go.opentelemetry.io/otel/metric v0.31.0 // indirect + go.opentelemetry.io/otel/sdk v1.10.0 // indirect + go.opentelemetry.io/otel/trace v1.10.0 // indirect + go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.19.0 // indirect - golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd // indirect - golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect + golang.org/x/crypto v0.1.0 // indirect + golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect golang.org/x/sys v0.3.0 // indirect - golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect - golang.org/x/text v0.3.7 // indirect + golang.org/x/term v0.3.0 // indirect + golang.org/x/text v0.5.0 // indirect golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect google.golang.org/api v0.60.0 // indirect google.golang.org/appengine v1.6.7 // indirect @@ -178,13 +174,15 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect - k8s.io/apiserver v0.25.0 // indirect - k8s.io/cloud-provider v0.25.0 // indirect - k8s.io/component-base v0.25.0 // indirect - k8s.io/component-helpers v0.25.0 // indirect + k8s.io/apiserver v0.26.0 // indirect + k8s.io/cloud-provider v0.26.0 // indirect + k8s.io/component-base v0.26.0 // indirect + k8s.io/component-helpers v0.26.0 // indirect k8s.io/cri-api v0.0.0 // indirect - k8s.io/csi-translation-lib v0.25.0 // indirect - k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect + k8s.io/csi-translation-lib v0.26.0 // indirect + k8s.io/dynamic-resource-allocation v0.26.0 // indirect + k8s.io/kms v0.26.0 // indirect + k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect k8s.io/kube-proxy v0.0.0 // indirect k8s.io/kube-scheduler v0.0.0 // indirect k8s.io/legacy-cloud-providers v0.0.0 // indirect @@ -198,29 +196,29 @@ require ( // need to override with commits (corresponding their kubernetes-* tags) replace ( github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2 - k8s.io/api => k8s.io/api v0.25.0 - k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.25.0 - k8s.io/apimachinery => k8s.io/apimachinery v0.25.0 - k8s.io/apiserver => k8s.io/apiserver v0.25.0 - k8s.io/cli-runtime => k8s.io/cli-runtime v0.25.0 - k8s.io/client-go => k8s.io/client-go v0.25.0 - k8s.io/cloud-provider => k8s.io/cloud-provider v0.25.0 - k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.25.0 - k8s.io/code-generator => k8s.io/code-generator v0.25.0 - k8s.io/component-base => k8s.io/component-base v0.25.0 - k8s.io/component-helpers => k8s.io/component-helpers v0.25.0 - k8s.io/controller-manager => k8s.io/controller-manager v0.25.0 - k8s.io/cri-api => k8s.io/cri-api v0.25.0 - k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.25.0 - k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.25.0 - k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.25.0 - k8s.io/kube-proxy => k8s.io/kube-proxy v0.25.0 - k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.25.0 - k8s.io/kubectl => k8s.io/kubectl v0.25.0 - k8s.io/kubelet => k8s.io/kubelet v0.25.0 - k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.25.0 - k8s.io/metrics => k8s.io/metrics v0.25.0 - k8s.io/mount-utils => k8s.io/mount-utils v0.25.0 - k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.25.0 - k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.25.0 + k8s.io/api => k8s.io/api v0.26.0 + k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.26.0 + k8s.io/apimachinery => k8s.io/apimachinery v0.26.0 + k8s.io/apiserver => k8s.io/apiserver v0.26.0 + k8s.io/cli-runtime => k8s.io/cli-runtime v0.26.0 + k8s.io/client-go => k8s.io/client-go v0.26.0 + k8s.io/cloud-provider => k8s.io/cloud-provider v0.26.0 + k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.26.0 + k8s.io/code-generator => k8s.io/code-generator v0.26.0 + k8s.io/component-base => k8s.io/component-base v0.26.0 + k8s.io/component-helpers => k8s.io/component-helpers v0.26.0 + k8s.io/controller-manager => k8s.io/controller-manager v0.26.0 + k8s.io/cri-api => k8s.io/cri-api v0.26.0 + k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.26.0 + k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.26.0 + k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.26.0 + k8s.io/kube-proxy => k8s.io/kube-proxy v0.26.0 + k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.26.0 + k8s.io/kubectl => k8s.io/kubectl v0.26.0 + k8s.io/kubelet => k8s.io/kubelet v0.26.0 + k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.26.0 + k8s.io/metrics => k8s.io/metrics v0.26.0 + k8s.io/mount-utils => k8s.io/mount-utils v0.26.0 + k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.26.0 + k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.26.0 ) diff --git a/go.sum b/go.sum index 46f4a4219b..60f8d38c7d 100644 --- a/go.sum +++ b/go.sum @@ -86,9 +86,7 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0 github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= @@ -98,7 +96,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed h1:ue9pVfIcP+QMEjfgo/Ez4ZjNZfonGgR6NgjMaJMu1Cg= +github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= +github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e h1:QEF07wC0T1rKkctt1RINW/+RMTVmiwxETico2l3gxJA= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -106,11 +105,9 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/auth0/go-jwt-middleware v1.0.1 h1:/fsQ4vRr4zod1wKReUH+0A3ySRjGiT9G34kypO/EKwI= github.com/aws/aws-sdk-go v1.35.24/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= -github.com/aws/aws-sdk-go v1.38.49 h1:E31vxjCe6a5I+mJLmUGaZobiWmg9KdWaud9IfceYeYQ= -github.com/aws/aws-sdk-go v1.38.49/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= +github.com/aws/aws-sdk-go v1.44.116 h1:NpLIhcvLWXJZAEwvPj3TDHeqp7DleK6ZUVYyW01WNHY= +github.com/aws/aws-sdk-go v1.44.116/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -121,7 +118,8 @@ github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdn github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= +github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= +github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -142,11 +140,12 @@ github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnht github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/container-storage-interface/spec v1.6.0 h1:vwN9uCciKygX/a0toYryoYD5+qI9ZFeAMuhEEKO+JBA= -github.com/container-storage-interface/spec v1.6.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s= +github.com/container-storage-interface/spec v1.7.0 h1:gW8eyFQUZWWrMWa8p1seJ28gwDoN5CVJ4uAbQ+Hdycw= +github.com/container-storage-interface/spec v1.7.0/go.mod h1:JYuzLqr9VVNoDJl44xp/8fmCOvWPDKzuGTwCoklhuqk= github.com/containerd/cgroups v1.0.1 h1:iJnMvco9XGvKUvNQkv88bE4uJXxRQH18efbKo9w5vHQ= github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= @@ -157,8 +156,9 @@ github.com/containerd/containerd v1.4.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMX github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= -github.com/containerd/ttrpc v1.0.2 h1:2/O3oTZN36q2xRolk0a2WWGgh7/Vf/liElg5hFYLX9U= github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= +github.com/containerd/ttrpc v1.1.0 h1:GbtyLRxb0gOLR0TYQWt3O6B0NvT8tMdorEHqIQo/lWI= +github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY= github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -173,7 +173,7 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI= @@ -183,23 +183,24 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v20.10.17+incompatible h1:JYCuMrWaVNophQTOrMMoSwudOVEfcegoZZrleKc1xwE= -github.com/docker/docker v20.10.17+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.18+incompatible h1:SN84VYXTBNGn92T/QwIRPlum9zfemfitN7pbsp26WSc= +github.com/docker/docker v20.10.18+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful/v3 v3.5.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/emicklei/go-restful/v3 v3.8.0 h1:eCZ8ulSerjdAiaNpF7GxXIE7ZCMo1moN1qX+S609eVw= github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= +github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -207,6 +208,7 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/euank/go-kmsg-parser v2.0.0+incompatible h1:cHD53+PLQuuQyLZeriD1V/esuG4MuU0Pjs5y6iknohY= @@ -215,16 +217,15 @@ github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= -github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= -github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= +github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= @@ -234,27 +235,30 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= -github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= +github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-ozzo/ozzo-validation v3.5.0+incompatible h1:sUy/in/P6askYr16XJgTKq/0SZhiWsdg4WZGaLsGQkM= -github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -270,6 +274,8 @@ github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzw github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8eU= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -309,9 +315,10 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/cadvisor v0.45.0 h1:bXQze1sd8srZiQwiQ19Qaq/AoMIZS8YceBXrIaEvkX0= -github.com/google/cadvisor v0.45.0/go.mod h1:vsMT3Uv2XjQ8M7WUtKARV74mU/HN64C4XtM1bJhUKcU= +github.com/google/cadvisor v0.46.0 h1:ryTIniqhN8/wR8UA1RuYSXHvsAtdpk/01XwTZtYHekY= +github.com/google/cadvisor v0.46.0/go.mod h1:YnCDnR8amaS0HoMEjheOI0TMPzFKCBLc30mciLEjwGI= github.com/google/cel-go v0.12.5 h1:DmzaiSgoaqGCjtpPQWl26/gND+yRpim56H1jCVev6d8= +github.com/google/cel-go v0.12.5/go.mod h1:Jk7ljRzLBhkmiAwBoUxB1sZSCVBAzkqPF25olK/iRDw= github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -326,8 +333,9 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -359,13 +367,9 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gax-go/v2 v2.1.1 h1:dp3bWCh+PPO1zjRRiCSczJav13sBvG4UhNyVTa1KqdU= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/gophercloud/gophercloud v0.1.0 h1:P/nh25+rzXouhytV2pUHBb65fnds26Ghl8/391+sT5o= -github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -378,19 +382,19 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/heketi/heketi v10.3.0+incompatible h1:X4DBFPzcyWZWhia32d94UhDECQJHH0M5kpRb1gxxUHk= -github.com/heketi/heketi v10.3.0+incompatible/go.mod h1:bB9ly3RchcQqsQ9CpyaQwvva7RS5ytVoSoholZQON6o= -github.com/heketi/tests v0.0.0-20151005000721-f3775cbcefd6 h1:oJ/NLadJn5HoxvonA6VxG31lg0d6XOURNA09BTtM4fY= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jaypipes/ghw v0.8.1-0.20210827132705-c7224150a17e h1:XTXPzmyiwx2uxk8JaU4mxmBZ+rzZtmEwkNm9H9ETzV0= github.com/jaypipes/ghw v0.8.1-0.20210827132705-c7224150a17e/go.mod h1:+gR9bjm3W/HnFi90liF+Fj9GpCe/Dsibl9Im8KmC7c4= github.com/jaypipes/pcidb v0.6.0 h1:VIM7GKVaW4qba30cvB67xSCgJPTzkG8Kzw/cbs5PHWU= @@ -418,8 +422,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k8stopologyawareschedwg/noderesourcetopology-api v0.0.12 h1:NhXbOjO1st8hIcVpegr3zw/AGG12vs3z//tCDDcfPpE= github.com/k8stopologyawareschedwg/noderesourcetopology-api v0.0.12/go.mod h1:AkACMQGiTgCt0lQw3m7TTU8PLH9lYKNK5e9DqFf5VuM= -github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= -github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI= +github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/cpuid/v2 v2.2.2 h1:xPMwiykqNK9VK0NYC3+jTMYv9I6Vl3YdjZgPZKG3zO0= @@ -441,15 +445,14 @@ github.com/libopenstorage/openstorage v1.0.0/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wn github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= -github.com/lpabon/godbc v0.1.1 h1:ilqjArN1UOENJJdM34I2YHKmF/B0gGq4VLoSGy9iAao= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= +github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989 h1:PS1dLCGtD8bb9RPKJrc8bS7qHL6JnW1CZvwzH9dPoUs= github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989/go.mod h1:2eu9pRWp8mo84xCg6KswZ+USQHjwgRhNp06sozOdsTY= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible h1:aKW/4cBs+yK6gpqU3K/oIwk9Q/XICqd3zOX/UFuvqmk= @@ -460,16 +463,15 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/ipvs v1.0.1 h1:aoZ7fhLTXgDbzVrAnvV+XbKOU8kOET7B3+xULDF/1o0= github.com/moby/ipvs v1.0.1/go.mod h1:2pngiyseZbIKXNv7hsKj3O9UEz30c53MT9005gt2hxQ= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= -github.com/moby/sys/mountinfo v0.6.0 h1:gUDhXQx58YNrpHlK4nSL+7y2pxFZkUcXqzFDKWdC0Oo= -github.com/moby/sys/mountinfo v0.6.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= -github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= -github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= +github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= +github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= +github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae h1:O4SWKdcHVCvYqyDV+9CJA1fcDN2L11Bule0iFy3YlAI= +github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -500,20 +502,25 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= -github.com/onsi/ginkgo/v2 v2.3.1 h1:8SbseP7qM32WcvE6VaN6vfXxv698izmsJ1UQX9ve7T8= -github.com/onsi/ginkgo/v2 v2.3.1/go.mod h1:Sv4yQXwG5VmF7tm3Q5Z+RWUpPo24LF1mpnz2crUb8Ys= +github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= +github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= +github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= +github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/onsi/gomega v1.22.1 h1:pY8O4lBfsHKZHM/6nrxkhVPUznOlIu3quZcKP/M20KI= +github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= +github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= +github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v1.1.3 h1:vIXrkId+0/J2Ymu2m7VjGvbSlAId9XNRPhn2p4b+d8w= -github.com/opencontainers/runc v1.1.3/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= +github.com/opencontainers/runc v1.1.4 h1:nRCz/8sKg6K6jgYAFLDlXzPeITBZJyX28DBVhWD+5dg= +github.com/opencontainers/runc v1.1.4/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 h1:3snG66yBm59tKhhSPQrQ/0bCrv1LQbKt40LnUPiUxdc= @@ -535,36 +542,41 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021 h1:if3/24+h9Sq6eDx8UUz1SO9cT9tizyIsATfB7b4D3tc= github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= -github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 h1:RpforrEYXWkmGwJHIGnLZ3tTWStkjVVstwzNGqxX2Ds= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= @@ -586,12 +598,11 @@ github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= -github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= +github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI= +github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -620,7 +631,6 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/negroni v1.0.0 h1:kIimOitoypq34K7TG7DUaJ9kq/N4Ofuwi1sjz0KipXc= github.com/vektra/errors v0.0.0-20140903201135-c64d83aba85a h1:lUVfiMMY/te9icPKBqOKkBIMZNxSpM90dxokDeCcfBg= github.com/vektra/errors v0.0.0-20140903201135-c64d83aba85a/go.mod h1:KUxJS71XlMs+ztT+RzsLRoWUQRUpECo/+Rb0EBk8/Wc= github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= @@ -643,16 +653,16 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= -go.etcd.io/etcd/api/v3 v3.5.4 h1:OHVyt3TopwtUQ2GKdd5wu3PmmipR4FTwCqoEjSyRdIc= -go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= -go.etcd.io/etcd/client/pkg/v3 v3.5.4 h1:lrneYvz923dvC14R54XcA7FXoZ3mlGZAgmwhfm7HqOg= -go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.4 h1:Dcx3/MYyfKcPNLpR4VVQUP5KgYrBeJtktBwEKkw08Ao= -go.etcd.io/etcd/client/v3 v3.5.4 h1:p83BUL3tAYS0OT/r0qglgc3M1JjhM0diV8DSWAhVXv4= -go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY= -go.etcd.io/etcd/pkg/v3 v3.5.4 h1:V5Dvl7S39ZDwjkKqJG2BfXgxZ3QREqqKifWQgIw5IM0= -go.etcd.io/etcd/raft/v3 v3.5.4 h1:YGrnAgRfgXloBNuqa+oBI/aRZMcK/1GS6trJePJ/Gqc= -go.etcd.io/etcd/server/v3 v3.5.4 h1:CMAZd0g8Bn5NRhynW6pKhc4FRg41/0QYy3d7aNm9874= +go.etcd.io/etcd/api/v3 v3.5.5 h1:BX4JIbQ7hl7+jL+g+2j5UAr0o1bctCm6/Ct+ArBGkf0= +go.etcd.io/etcd/api/v3 v3.5.5/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= +go.etcd.io/etcd/client/pkg/v3 v3.5.5 h1:9S0JUVvmrVl7wCF39iTQthdaaNIiAaQbmK75ogO6GU8= +go.etcd.io/etcd/client/pkg/v3 v3.5.5/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ= +go.etcd.io/etcd/client/v2 v2.305.5 h1:DktRP60//JJpnPC0VBymAN/7V71GHMdjDCBt4ZPXDjI= +go.etcd.io/etcd/client/v3 v3.5.5 h1:q++2WTJbUgpQu4B6hCuT7VkdwaTP7Qz6Daak3WzbrlI= +go.etcd.io/etcd/client/v3 v3.5.5/go.mod h1:aApjR4WGlSumpnJ2kloS75h6aHUmAyaPLjHMxpc7E7c= +go.etcd.io/etcd/pkg/v3 v3.5.5 h1:Ablg7T7OkR+AeeeU32kdVhw/AGDsitkKPl7aW73ssjU= +go.etcd.io/etcd/raft/v3 v3.5.5 h1:Ibz6XyZ60OYyRopu73lLM/P+qco3YtlZMOhnXNS051I= +go.etcd.io/etcd/server/v3 v3.5.5 h1:jNjYm/9s+f9A9r6+SC4RvNaz6AqixpOvhrFdT0PvIj0= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -661,34 +671,30 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/contrib v0.20.0 h1:ubFQUn0VCZ0gPwIoJfBJVpeBlyRMxu8Mm/huKWYd9p0= -go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= -go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.20.0 h1:8YW+SL62UmcwRQJFZVfnyOlIUUtmlR13NaMKi+Fa6Fo= -go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.20.0/go.mod h1:oQkZOyq61qZBItEFqhfpobK6X/oDPR7/Qr+MXjVSTks= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 h1:sO4WKdPAudZGKPcpZT4MJn6JaDmpyLrMPDGGyA1SttE= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0 h1:Q3C9yzW6I9jqEc8sawxzxZmY48fs9u220KXq6d5s3XU= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= -go.opentelemetry.io/contrib/propagators v0.20.0 h1:IrLQng5Z7AfzkS4sEsYaj2ejkO4FCkgKdAr1aYKOfNc= -go.opentelemetry.io/contrib/propagators v0.20.0/go.mod h1:yLmt93MeSiARUwrK57bOZ4FBruRN4taLiW1lcGfnOes= -go.opentelemetry.io/otel v0.20.0 h1:eaP0Fqu7SXHwvjiqDq83zImeehOHX8doTvU9AwXON8g= -go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= -go.opentelemetry.io/otel/exporters/otlp v0.20.0 h1:PTNgq9MRmQqqJY0REVbZFvwkYOA85vbdQU/nVfxDyqg= -go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= -go.opentelemetry.io/otel/metric v0.20.0 h1:4kzhXFP+btKm4jwxpjIqjs41A7MakRFUS86bqLHTIw8= -go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/oteltest v0.20.0 h1:HiITxCawalo5vQzdHfKeZurV8x7ljcqAgiWzF6Vaeaw= -go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= -go.opentelemetry.io/otel/sdk v0.20.0 h1:JsxtGXd06J8jrnya7fdI/U/MR6yXA5DtbZy+qoHQlr8= -go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/sdk/export/metric v0.20.0 h1:c5VRjxCXdQlx1HjzwGdQHzZaVI82b5EbBgOu2ljD92g= -go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= -go.opentelemetry.io/otel/sdk/metric v0.20.0 h1:7ao1wpzHRVKf0OQ7GIxiQJA6X7DLX9o14gmVon7mMK8= -go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= -go.opentelemetry.io/otel/trace v0.20.0 h1:1DL6EXUdcg95gukhuRRvLDO/4X5THh/5dIV52lqtnbw= -go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= -go.opentelemetry.io/proto/otlp v0.7.0 h1:rwOQPCuKAKmwGKq2aVNnYIibI6wnV7EvzgfTCzcdGg8= +go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.35.0 h1:KQjX0qQ8H21oBUAvFp4ZLKJMMLIluONvSPDAFIGmX58= +go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.35.0/go.mod h1:DQYkU9srMFqLUTVA/7/WlRHdnYDB7wyMMlle2ktMjfI= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 h1:xFSRQBbXF6VvYRf2lqMJXxoB72XI1K/azav8TekHHSw= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0/go.mod h1:h8TWwRAhQpOd0aM5nYsRD8+flnkj+526GEIVlarH7eY= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0 h1:Ajldaqhxqw/gNzQA45IKFWLdG7jZuXX/wBW1d5qvbUI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0/go.mod h1:9NiG9I2aHTKkcxqCILhjtyNA1QEiCjdBACv4IvrFQ+c= +go.opentelemetry.io/contrib/propagators/b3 v1.10.0 h1:6AD2VV8edRdEYNaD8cNckpzgdMLU2kbV9OYyxt2kvCg= +go.opentelemetry.io/otel v1.10.0 h1:Y7DTJMR6zs1xkS/upamJYk0SxxN4C9AqRd77jmZnyY4= +go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 h1:TaB+1rQhddO1sF71MpZOZAuSPW1klK2M8XxfrBMfK7Y= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0/go.mod h1:78XhIg8Ht9vR4tbLNUhXsiOnE2HOuSeKAiAcoVQEpOY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 h1:pDDYmo0QadUPal5fwXoY1pmMpFcdyhXOmL5drCrI3vU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0/go.mod h1:Krqnjl22jUJ0HgMzw5eveuCvFDXY4nSYb4F8t5gdrag= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 h1:KtiUEhQmj/Pa874bVYKGNVdq8NPKiacPbaRRtgXi+t4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0/go.mod h1:OfUCyyIiDvNXHWpcWgbF+MWvqPZiNa3YDEnivcnYsV0= +go.opentelemetry.io/otel/metric v0.31.0 h1:6SiklT+gfWAwWUR0meEMxQBtihpiEs4c+vL9spDTqUs= +go.opentelemetry.io/otel/metric v0.31.0/go.mod h1:ohmwj9KTSIeBnDBm/ZwH2PSZxZzoOaG2xZeekTRzL5A= +go.opentelemetry.io/otel/sdk v1.10.0 h1:jZ6K7sVn04kk/3DNUdJ4mqRlGDiXAVuIG+MMENpTNdY= +go.opentelemetry.io/otel/sdk v1.10.0/go.mod h1:vO06iKzD5baltJz1zarxMCNHFpUlUiOy4s65ECtn6kE= +go.opentelemetry.io/otel/trace v1.10.0 h1:npQMbR8o7mum8uF95yFbOEJffhs1sbCOfDh8zAJiH5E= +go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/AzrK+kxfGqySM= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= +go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= @@ -703,7 +709,6 @@ go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -711,8 +716,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd h1:XcWmESyNjXJMLahc3mqVQJcgSTDxFxhETVlfk9uGc38= -golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -750,6 +755,7 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -797,9 +803,13 @@ golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10 h1:Frnccbp+ok2GkUS2tC84yAq/U9Vg+0sIO7aRL3T4Xnc= +golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -816,8 +826,9 @@ golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 h1:RerP+noqYHUQ8CMRcPlC2nvTa4dcBIjegkuWdcUDuqg= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b h1:clP8eMhB30EHdc0bd2Twtq6kgU7yl5ub2cQLSdrv1Dg= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -829,6 +840,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -836,7 +848,6 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -908,14 +919,19 @@ golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -924,8 +940,10 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -991,6 +1009,7 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1093,6 +1112,7 @@ google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEc google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211021150943-2b146023228c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 h1:hrbNEivu7Zn1pxvHk6MBrq9iE22woVILTHqexqBxe6I= google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -1121,9 +1141,11 @@ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8= -google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.49.0 h1:WTLtQzmQori5FUH25Pq4WT22oCsv8USpQ+F6rqtsmxw= +google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1138,8 +1160,9 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1186,61 +1209,65 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= -k8s.io/api v0.25.0 h1:H+Q4ma2U/ww0iGB78ijZx6DRByPz6/733jIuFpX70e0= -k8s.io/api v0.25.0/go.mod h1:ttceV1GyV1i1rnmvzT3BST08N6nGt+dudGrquzVQWPk= -k8s.io/apiextensions-apiserver v0.25.0 h1:CJ9zlyXAbq0FIW8CD7HHyozCMBpDSiH7EdrSTCZcZFY= -k8s.io/apiextensions-apiserver v0.25.0/go.mod h1:3pAjZiN4zw7R8aZC5gR0y3/vCkGlAjCazcg1me8iB/E= -k8s.io/apimachinery v0.25.0 h1:MlP0r6+3XbkUG2itd6vp3oxbtdQLQI94fD5gCS+gnoU= -k8s.io/apimachinery v0.25.0/go.mod h1:qMx9eAk0sZQGsXGu86fab8tZdffHbwUfsvzqKn4mfB0= -k8s.io/apiserver v0.25.0 h1:8kl2ifbNffD440MyvHtPaIz1mw4mGKVgWqM0nL+oyu4= -k8s.io/apiserver v0.25.0/go.mod h1:BKwsE+PTC+aZK+6OJQDPr0v6uS91/HWxX7evElAH6xo= -k8s.io/cli-runtime v0.25.0 h1:XBnTc2Fi+w818jcJGzhiJKQuXl8479sZ4FhtV5hVJ1Q= -k8s.io/client-go v0.25.0 h1:CVWIaCETLMBNiTUta3d5nzRbXvY5Hy9Dpl+VvREpu5E= -k8s.io/client-go v0.25.0/go.mod h1:lxykvypVfKilxhTklov0wz1FoaUZ8X4EwbhS6rpRfN8= -k8s.io/cloud-provider v0.25.0 h1:ONX5BON6f1Mxa2GWvPyKn+QsZXaLauPUte7MZxfWUro= -k8s.io/cloud-provider v0.25.0/go.mod h1:afVfVCIYOUER914WmSp0QpAtJn12gv4qu9NMT4XBxZo= -k8s.io/cluster-bootstrap v0.25.0 h1:KJ2/r0dV+bLfTK5EBobAVKvjGel3N4Qqh3bvnzh9qPk= -k8s.io/code-generator v0.25.0/go.mod h1:B6jZgI3DvDFAualltPitbYMQ74NjaCFxum3YeKZZ+3w= -k8s.io/component-base v0.25.0 h1:haVKlLkPCFZhkcqB6WCvpVxftrg6+FK5x1ZuaIDaQ5Y= -k8s.io/component-base v0.25.0/go.mod h1:F2Sumv9CnbBlqrpdf7rKZTmmd2meJq0HizeyY/yAFxk= -k8s.io/component-helpers v0.25.0 h1:vNzYfqnVXj7f+CPksduKVv2Z9kC+IDsOs9yaOyxZrj0= -k8s.io/component-helpers v0.25.0/go.mod h1:auaFj2bvb5Zmy0mLk4WJNmwP0w4e7Zk+/Tu9FFBGA20= -k8s.io/cri-api v0.25.0 h1:INwdXsCDSA/0hGNdPxdE2dQD6ft/5K1EaKXZixvSQxg= -k8s.io/cri-api v0.25.0/go.mod h1:J1rAyQkSJ2Q6I+aBMOVgg2/cbbebso6FNa0UagiR0kc= -k8s.io/csi-translation-lib v0.25.0 h1:Jh3kn5p3kEGGA/q1fovTNIG9fypzt2c34sm+qij2W/8= -k8s.io/csi-translation-lib v0.25.0/go.mod h1:Wb80CDywP4753F6wWkIyOuJIQtQAbhgw985veSgAn/4= +k8s.io/api v0.26.0 h1:IpPlZnxBpV1xl7TGk/X6lFtpgjgntCg8PJ+qrPHAC7I= +k8s.io/api v0.26.0/go.mod h1:k6HDTaIFC8yn1i6pSClSqIwLABIcLV9l5Q4EcngKnQg= +k8s.io/apiextensions-apiserver v0.26.0 h1:Gy93Xo1eg2ZIkNX/8vy5xviVSxwQulsnUdQ00nEdpDo= +k8s.io/apiextensions-apiserver v0.26.0/go.mod h1:7ez0LTiyW5nq3vADtK6C3kMESxadD51Bh6uz3JOlqWQ= +k8s.io/apimachinery v0.26.0 h1:1feANjElT7MvPqp0JT6F3Ss6TWDwmcjLypwoPpEf7zg= +k8s.io/apimachinery v0.26.0/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= +k8s.io/apiserver v0.26.0 h1:q+LqIK5EZwdznGZb8bq0+a+vCqdeEEe4Ux3zsOjbc4o= +k8s.io/apiserver v0.26.0/go.mod h1:aWhlLD+mU+xRo+zhkvP/gFNbShI4wBDHS33o0+JGI84= +k8s.io/cli-runtime v0.26.0 h1:aQHa1SyUhpqxAw1fY21x2z2OS5RLtMJOCj7tN4oq8mw= +k8s.io/client-go v0.26.0 h1:lT1D3OfO+wIi9UFolCrifbjUUgu7CpLca0AD8ghRLI8= +k8s.io/client-go v0.26.0/go.mod h1:I2Sh57A79EQsDmn7F7ASpmru1cceh3ocVT9KlX2jEZg= +k8s.io/cloud-provider v0.26.0 h1:kO2BIgCou71QNRHGkpFi/8lnas9UIr+fJz1l/nuiOMo= +k8s.io/cloud-provider v0.26.0/go.mod h1:JwfUAH67C8f7t6tOC4v4ty+DuvIYVjNF6bGVYSDCqqs= +k8s.io/cluster-bootstrap v0.26.0 h1:jd6T3WmpZo6TpmIHqg1wc4bX/BLsGC8Tzle/VKI9vRo= +k8s.io/code-generator v0.26.0/go.mod h1:OMoJ5Dqx1wgaQzKgc+ZWaZPfGjdRq/Y3WubFrZmeI3I= +k8s.io/component-base v0.26.0 h1:0IkChOCohtDHttmKuz+EP3j3+qKmV55rM9gIFTXA7Vs= +k8s.io/component-base v0.26.0/go.mod h1:lqHwlfV1/haa14F/Z5Zizk5QmzaVf23nQzCwVOQpfC8= +k8s.io/component-helpers v0.26.0 h1:KNgwqs3EUdK0HLfW4GhnbD+q/Zl9U021VfIU7qoVYFk= +k8s.io/component-helpers v0.26.0/go.mod h1:jHN01qS/Jdj95WCbTe9S2VZ9yxpxXNY488WjF+yW4fo= +k8s.io/cri-api v0.26.0 h1:/Cfs9BUtGwYWjRCscd/4q+uJ0UqCzwcIZDI+Eyvle78= +k8s.io/cri-api v0.26.0/go.mod h1:I5TGOn/ziMzqIcUvsYZzVE8xDAB1JBkvcwvR0yDreuw= +k8s.io/csi-translation-lib v0.26.0 h1:bCvlfw53Kmyn7cvXeYGe9aqqzR1b0xrGs2XEWHFW+es= +k8s.io/csi-translation-lib v0.26.0/go.mod h1:zRKLRqER6rA8NCKQBhVIdkyDHKgNlu2BK1RKTHjcw+8= +k8s.io/dynamic-resource-allocation v0.26.0 h1:zljrsqa0PxrIwNklTnGBA/az6+33SUQwsNNNKdVTzwg= +k8s.io/dynamic-resource-allocation v0.26.0/go.mod h1:K+hO5A+QsSknRjlhfbUtvZVYUblOldvYyT51eGrZyWI= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20220902162205-c0856e24416d/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ= -k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-aggregator v0.25.0 h1:T0AZZh5nkEcXJnK1o87IghRCZf/KDtzJQWIPulQQlHU= -k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 h1:MQ8BAZPZlWk3S9K4a9NCkIFQtZShWqoha7snGixVgEA= -k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU= -k8s.io/kube-proxy v0.25.0 h1:QuoKEyXV+NNMXEh8oqlthUlHkmWF+WBnYUMHCf817k0= -k8s.io/kube-proxy v0.25.0/go.mod h1:uHv1HwMVDYgl1pU2PTDKLRlxtNOf4z2M5YPYC6NP1CU= -k8s.io/kube-scheduler v0.25.0 h1:Up2rW+1H3JsgcpfdMcj/kVbYtgoxpiwxKLg5L4PaZ98= -k8s.io/kube-scheduler v0.25.0/go.mod h1:cwiyJeImgFbhmbnImzvuhbiJayNngRNEe3FJkZDPw9Y= -k8s.io/kubectl v0.25.0 h1:/Wn1cFqo8ik3iee1EvpxYre3bkWsGLXzLQI6uCCAkQc= -k8s.io/kubectl v0.25.0/go.mod h1:n16ULWsOl2jmQpzt2o7Dud1t4o0+Y186ICb4O+GwKAU= -k8s.io/kubelet v0.25.0 h1:eTS5B1u1o63ndExAHKLJytzz/GBy86ROcxYtu0VK3RA= -k8s.io/kubelet v0.25.0/go.mod h1:J6aQxrZdSsGPrskYrhZdEn6PCnGha+GNvF0g9aWfQnw= -k8s.io/kubernetes v1.25.3 h1:Ljx/Ew9+dt7rN9ob3V+N/aoDy7nDSbmr35IbYGRTyqE= -k8s.io/kubernetes v1.25.3/go.mod h1:lvEY+3iJhh+sGIK1LorGkI56rW0eLGsfalnp68wQwYU= -k8s.io/legacy-cloud-providers v0.25.0 h1:c+boKaCw/2ZzvA8/XgTIeDrGUJJ2Ucy+jLJFf02+huE= -k8s.io/legacy-cloud-providers v0.25.0/go.mod h1:bnmUgHHeBmK3M9JgQzu+ne6UCUVURDzkpF0Y7VeypVE= -k8s.io/metrics v0.25.0 h1:z/tyqXUCxvmFsKIO7GH6ulvogYvGp+pDmlz5ANSQVPE= -k8s.io/mount-utils v0.25.0 h1:dx+SKXBVjskPgkpv9Mk0mAfbLNOxz8jAqTXGTZJnd8I= -k8s.io/mount-utils v0.25.0/go.mod h1:WTYq8Ev/JrnkqK2h1jFUnC8qWGuqzMb9XDC+Lu3WNU0= -k8s.io/pod-security-admission v0.25.0 h1:Sceq45pO7E7RTaYAr3Br94ZMDISJIngvXXcAfcZJufk= -k8s.io/pod-security-admission v0.25.0/go.mod h1:b/UC586Th2LijoNV+ssyyAryUvmaTrEWms5ZzBEkVsA= -k8s.io/sample-apiserver v0.25.0 h1:YvpJa31xRnNzFfkiF9/8gT0AzxMWwGlamT/aEClhHPo= +k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= +k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kms v0.26.0 h1:5+GOQLvUajSd0z5ODF52RzB2rHo1HJUSYsVC3Ri3VgI= +k8s.io/kms v0.26.0/go.mod h1:ReC1IEGuxgfN+PDCIpR6w8+XMmDE7uJhxcCwMZFdIYc= +k8s.io/kube-aggregator v0.26.0 h1:XF/Q5FwdLmCsK1RKGFNWfIo/b+r63sXOu+KKcaIFa/M= +k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= +k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= +k8s.io/kube-proxy v0.26.0 h1:VBC83bWr5L4GKSxRFz0YBbwGgQITc0+p8avGzw0LNKo= +k8s.io/kube-proxy v0.26.0/go.mod h1:4kz3dPdMUnspJnFgoJG9lWn1UCiho85Gyn1WLInK0XA= +k8s.io/kube-scheduler v0.26.0 h1:PjSF4cF9X7cAMj5MZ9ZSq2RJ2VkcKKCKj6fy/EbxtA0= +k8s.io/kube-scheduler v0.26.0/go.mod h1:FmptJbq36ATKYxeR+UqAvUtFaLeoFWgoDk1cdCpVPYQ= +k8s.io/kubectl v0.26.0 h1:xmrzoKR9CyNdzxBmXV7jW9Ln8WMrwRK6hGbbf69o4T0= +k8s.io/kubectl v0.26.0/go.mod h1:eInP0b+U9XUJWSYeU9XZnTA+cVYuWyl3iYPGtru0qhQ= +k8s.io/kubelet v0.26.0 h1:08bDb5IoUH/1K1t2NUwnGIIWxjm9LSqn6k3FWw1tJGI= +k8s.io/kubelet v0.26.0/go.mod h1:DluF+d8jS2nE/Hs7CC3QM+OZlIEb22NTOihQ3EDwCQ4= +k8s.io/kubernetes v1.26.0 h1:fL8VMr4xlfTazPORLhz5fsvO5I3bsFpmynVxZTH1ItQ= +k8s.io/kubernetes v1.26.0/go.mod h1:z0aCJwn6DxzB/dDiWLbQaJO5jWOR2qoaCMnmSAx45XM= +k8s.io/legacy-cloud-providers v0.26.0 h1:PudIbGlvQLwSkden/vSfRE4YtsyKCRjQs0OAiP1w8M8= +k8s.io/legacy-cloud-providers v0.26.0/go.mod h1:dOOgYhHiMNWNla/XyM4Ppgjcrn3HulGa93/0vUO5/z4= +k8s.io/metrics v0.26.0 h1:U/NzZHKDrIVGL93AUMRkqqXjOah3wGvjSnKmG/5NVCs= +k8s.io/mount-utils v0.26.0 h1:MG5oXE2aF1UHMJ3KFbVtBtiRA4J/2u0sijrkfsoaMwU= +k8s.io/mount-utils v0.26.0/go.mod h1:au99w4FWU5ZWelLb3Yx6kJc8RZ387IyWVM9tN65Yhxo= +k8s.io/pod-security-admission v0.26.0 h1:XBG/uyP2cYwSFr5IWAQ1IIArxMYARJKzEzSmP4ZbC1s= +k8s.io/pod-security-admission v0.26.0/go.mod h1:HQHvpCrn6KQLKRUqFvWkHCVKet3X62fn2F3j5anYiEM= +k8s.io/sample-apiserver v0.26.0 h1:8ackiudVsxJhfdUxIw+ftG14+5AYr51RYhcO/+xeZYY= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4= -k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20221107191617-1a15be271d1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y= +k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= @@ -1252,5 +1279,6 @@ sigs.k8s.io/kustomize/api v0.12.1 h1:7YM7gW3kYBwtKvoY216ZzY+8hM+lV53LUayghNRJ0vM sigs.k8s.io/kustomize/kyaml v0.13.9 h1:Qz53EAaFFANyNgyOEJbT/yoIHygK40/ZcvU3rgry2Tk= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index a11a1f0e53..68f97a1516 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -34,7 +34,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" - e2elog "k8s.io/kubernetes/test/e2e/framework/log" + e2elog "k8s.io/kubernetes/test/e2e/framework" e2enetwork "k8s.io/kubernetes/test/e2e/framework/network" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" diff --git a/test/e2e/utils/config.go b/test/e2e/utils/config.go index ab8b3dbfce..2620fdac7d 100644 --- a/test/e2e/utils/config.go +++ b/test/e2e/utils/config.go @@ -21,7 +21,7 @@ import ( "os" "regexp" - e2elog "k8s.io/kubernetes/test/e2e/framework/log" + e2elog "k8s.io/kubernetes/test/e2e/framework" "sigs.k8s.io/yaml" ) From b296bdf0b3f4182c942b4ce8c3905c9c63718e8e Mon Sep 17 00:00:00 2001 From: Feruzjon Muyassarov Date: Mon, 12 Dec 2022 22:47:09 +0200 Subject: [PATCH 40/87] update test functions according to upstream deprecated/removed methods Signed-off-by: Feruzjon Muyassarov --- pkg/nfd-master/nfd-api-controller.go | 2 +- pkg/resourcemonitor/excludelist.go | 5 +++-- test/e2e/node_feature_discovery.go | 2 +- test/e2e/topology_updater.go | 16 +++++++++------- test/e2e/utils/pod/pod.go | 10 ++++++---- 5 files changed, 20 insertions(+), 15 deletions(-) diff --git a/pkg/nfd-master/nfd-api-controller.go b/pkg/nfd-master/nfd-api-controller.go index 04212b137a..a2407efc2a 100644 --- a/pkg/nfd-master/nfd-api-controller.go +++ b/pkg/nfd-master/nfd-api-controller.go @@ -46,7 +46,7 @@ func newNfdController(config *restclient.Config) *nfdController { informerFactory := nfdinformers.NewSharedInformerFactory(nfdClient, 5*time.Minute) ruleInformer := informerFactory.Nfd().V1alpha1().NodeFeatureRules() - ruleInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + _, _ = ruleInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(object interface{}) { key, _ := cache.MetaNamespaceKeyFunc(object) klog.V(2).Infof("NodeFeatureRule %v added", key) diff --git a/pkg/resourcemonitor/excludelist.go b/pkg/resourcemonitor/excludelist.go index 444042f469..590d0bb8ab 100644 --- a/pkg/resourcemonitor/excludelist.go +++ b/pkg/resourcemonitor/excludelist.go @@ -8,12 +8,13 @@ import ( // ExcludeResourceList contains a list of resources to ignore during resources scan type ExcludeResourceList struct { - excludeList sets.String + excludeList sets.Set[string] } // NewExcludeResourceList returns new ExcludeList with values with set.String types func NewExcludeResourceList(resMap map[string][]string, nodeName string) ExcludeResourceList { - excludeList := make(sets.String) + excludeList := make(sets.Set[string]) + for k, v := range resMap { if k == nodeName || k == "*" { excludeList.Insert(v...) diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index 68f97a1516..39c50ccad3 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -115,7 +115,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { // Launch nfd-master By("Creating nfd master pod and nfd-master service") imageOpt := testpod.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)) - masterPod = f.PodClient().CreateSync(testpod.NFDMaster(imageOpt)) + masterPod = e2epod.NewPodClient(f).CreateSync(testpod.NFDMaster(imageOpt)) // Create nfd-master service nfdSvc, err := testutils.CreateService(f.ClientSet, f.Namespace.Name) diff --git a/test/e2e/topology_updater.go b/test/e2e/topology_updater.go index fb9ec48220..af69919948 100644 --- a/test/e2e/topology_updater.go +++ b/test/e2e/topology_updater.go @@ -23,6 +23,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" + "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -39,6 +40,7 @@ import ( "k8s.io/kubernetes/test/e2e/framework/kubelet" admissionapi "k8s.io/pod-security-admission/api" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" testutils "sigs.k8s.io/node-feature-discovery/test/e2e/utils" testds "sigs.k8s.io/node-feature-discovery/test/e2e/utils/daemonset" testpod "sigs.k8s.io/node-feature-discovery/test/e2e/utils/pod" @@ -97,16 +99,16 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { Expect(err).NotTo(HaveOccurred()) }) - // TODO: replace with regular AfterEach once we have https://github.com/kubernetes/kubernetes/pull/111998 in - f.AddAfterEach("Node Feature Discovery topology updater CRD and RBAC removal", func(f *framework.Framework, failed bool) { + ginkgo.AfterEach(func() { + framework.Logf("Node Feature Discovery topology updater CRD and RBAC removal") err := testutils.DeconfigureRBAC(f.ClientSet, f.Namespace.Name) if err != nil { - framework.Logf("failed to delete RBAC resources: %v", err) + framework.Failf("AfterEach: Failed to delete RBAC resources: %v", err) } }) Context("with topology-updater daemonset running", func() { - BeforeEach(func() { + ginkgo.BeforeEach(func() { cfg, err := testutils.GetConfig() Expect(err).ToNot(HaveOccurred()) @@ -130,7 +132,7 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { sleeperPod := testpod.BestEffortSleeper() podMap := make(map[string]*corev1.Pod) - pod := f.PodClient().CreateSync(sleeperPod) + pod := e2epod.NewPodClient(f).CreateSync(sleeperPod) podMap[pod.Name] = pod defer testpod.DeleteAsync(f, podMap) @@ -175,7 +177,7 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { })) podMap := make(map[string]*corev1.Pod) - pod := f.PodClient().CreateSync(sleeperPod) + pod := e2epod.NewPodClient(f).CreateSync(sleeperPod) podMap[pod.Name] = pod defer testpod.DeleteAsync(f, podMap) @@ -230,7 +232,7 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { sleeperPod.Spec.NodeName = topologyUpdaterNode.Name podMap := make(map[string]*corev1.Pod) - pod := f.PodClient().CreateSync(sleeperPod) + pod := e2epod.NewPodClient(f).CreateSync(sleeperPod) podMap[pod.Name] = pod defer testpod.DeleteAsync(f, podMap) diff --git a/test/e2e/utils/pod/pod.go b/test/e2e/utils/pod/pod.go index 0e92c0df76..674f38e6c5 100644 --- a/test/e2e/utils/pod/pod.go +++ b/test/e2e/utils/pod/pod.go @@ -31,7 +31,9 @@ import ( "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubectl/pkg/util/podutils" + "k8s.io/kubernetes/test/e2e/framework" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" "k8s.io/utils/pointer" "sigs.k8s.io/node-feature-discovery/test/e2e/utils" @@ -111,7 +113,7 @@ func DeleteSyncByName(f *framework.Framework, podName string) { delOpts := metav1.DeleteOptions{ GracePeriodSeconds: &gp, } - f.PodClient().DeleteSync(podName, delOpts, framework.DefaultPodDeletionTimeout) + e2epod.NewPodClient(f).DeleteSync(podName, delOpts, e2epod.DefaultPodDeletionTimeout) } type SpecOption func(spec *corev1.PodSpec) @@ -382,9 +384,9 @@ func NFDTopologyUpdaterSpec(kc utils.KubeletConfig, opts ...SpecOption) *corev1. Capabilities: &corev1.Capabilities{ Drop: []corev1.Capability{"ALL"}, }, - RunAsUser: pointer.Int64Ptr(0), - ReadOnlyRootFilesystem: pointer.BoolPtr(true), - AllowPrivilegeEscalation: pointer.BoolPtr(false), + RunAsUser: pointer.Int64(0), + ReadOnlyRootFilesystem: pointer.Bool(true), + AllowPrivilegeEscalation: pointer.Bool(false), }, VolumeMounts: []corev1.VolumeMount{ { From 079655b42cb5c28e4701abc34a579f870dafb601 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Wed, 14 Dec 2022 00:27:27 +0200 Subject: [PATCH 41/87] nfd-master: add error checking for CRD controller creation --- pkg/nfd-master/nfd-api-controller.go | 10 ++++++---- pkg/nfd-master/nfd-master.go | 5 ++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/pkg/nfd-master/nfd-api-controller.go b/pkg/nfd-master/nfd-api-controller.go index a2407efc2a..4a2acc46a8 100644 --- a/pkg/nfd-master/nfd-api-controller.go +++ b/pkg/nfd-master/nfd-api-controller.go @@ -37,7 +37,7 @@ type nfdController struct { stopChan chan struct{} } -func newNfdController(config *restclient.Config) *nfdController { +func newNfdController(config *restclient.Config) (*nfdController, error) { c := &nfdController{ stopChan: make(chan struct{}, 1), } @@ -46,7 +46,7 @@ func newNfdController(config *restclient.Config) *nfdController { informerFactory := nfdinformers.NewSharedInformerFactory(nfdClient, 5*time.Minute) ruleInformer := informerFactory.Nfd().V1alpha1().NodeFeatureRules() - _, _ = ruleInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + if _, err := ruleInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(object interface{}) { key, _ := cache.MetaNamespaceKeyFunc(object) klog.V(2).Infof("NodeFeatureRule %v added", key) @@ -59,14 +59,16 @@ func newNfdController(config *restclient.Config) *nfdController { key, _ := cache.MetaNamespaceKeyFunc(object) klog.V(2).Infof("NodeFeatureRule %v deleted", key) }, - }) + }); err != nil { + return nil, err + } informerFactory.Start(c.stopChan) utilruntime.Must(nfdv1alpha1.AddToScheme(nfdscheme.Scheme)) c.ruleLister = ruleInformer.Lister() - return c + return c, nil } func (c *nfdController) stop() { diff --git a/pkg/nfd-master/nfd-master.go b/pkg/nfd-master/nfd-master.go index 29c88bbd0b..db4f9f38e2 100644 --- a/pkg/nfd-master/nfd-master.go +++ b/pkg/nfd-master/nfd-master.go @@ -155,7 +155,10 @@ func (m *nfdMaster) Run() error { return err } klog.Info("starting nfd api controller") - m.nfdController = newNfdController(kubeconfig) + m.nfdController, err = newNfdController(kubeconfig) + if err != nil { + return fmt.Errorf("failed to initialize CRD controller: %w", err) + } } if !m.args.NoPublish { From 59ebff46c9a249dd21f4c3f3b805e46cda172530 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Mon, 4 Jul 2022 17:59:53 +0300 Subject: [PATCH 42/87] apis/nfd: add CRD for communicating node features Add a new NodeFeature CRD to the nfd Kubernetes API to communicate node features over K8s api objects instead of gRPC. The new resource is namespaced which will help the management of multiple NodeFeature objects per node. This aims at enabling 3rd party detectors for custom features. In addition to communicating raw features the NodeFeature object also has a field for directly requesting labels that should be applied on the node object. Rename the crd deployment file to nfd-api-crds.yaml so that it matches the new content of the file. Also, rename the Helm subdir for CRDs to match the expected chart directory structure. --- deployment/base/nfd-crds/kustomization.yaml | 2 +- hack/generate.sh | 4 +-- pkg/apis/nfd/v1alpha1/annotations_labels.go | 6 ++++ pkg/apis/nfd/v1alpha1/register.go | 1 + pkg/apis/nfd/v1alpha1/types.go | 31 +++++++++++++++++++++ 5 files changed, 41 insertions(+), 3 deletions(-) diff --git a/deployment/base/nfd-crds/kustomization.yaml b/deployment/base/nfd-crds/kustomization.yaml index 73fd8cc08d..b7f95447ee 100644 --- a/deployment/base/nfd-crds/kustomization.yaml +++ b/deployment/base/nfd-crds/kustomization.yaml @@ -2,4 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- nodefeaturerule-crd.yaml +- nfd-api-crds.yaml diff --git a/hack/generate.sh b/hack/generate.sh index 9b3b5ca222..0d5c165d61 100755 --- a/hack/generate.sh +++ b/hack/generate.sh @@ -10,10 +10,10 @@ go generate ./cmd/... ./pkg/... ./source/... rm -rf vendor/ -controller-gen object crd output:crd:stdout paths=./pkg/apis/... > deployment/base/nfd-crds/nodefeaturerule-crd.yaml +controller-gen object crd output:crd:stdout paths=./pkg/apis/... > deployment/base/nfd-crds/nfd-api-crds.yaml mkdir -p deployment/helm/node-feature-discovery/crds -cp deployment/base/nfd-crds/nodefeaturerule-crd.yaml deployment/helm/node-feature-discovery/crds/ +cp deployment/base/nfd-crds/nfd-api-crds.yaml deployment/helm/node-feature-discovery/crds rm -rf sigs.k8s.io diff --git a/pkg/apis/nfd/v1alpha1/annotations_labels.go b/pkg/apis/nfd/v1alpha1/annotations_labels.go index 7e4cbee250..978ba853c4 100644 --- a/pkg/apis/nfd/v1alpha1/annotations_labels.go +++ b/pkg/apis/nfd/v1alpha1/annotations_labels.go @@ -46,4 +46,10 @@ const ( // NodeTaintsAnnotation is the annotation that holds the taints that nfd-master set on the node NodeTaintsAnnotation = AnnotationNs + "/taints" + + // NodeFeatureObjNodeNameLabel is the label that specifies which node the + // NodeFeature object is targeting. Creators of NodeFeature objects must + // set this label and consumers of the objects are supposed to use the + // label for filtering features designated for a certain node. + NodeFeatureObjNodeNameLabel = "nfd.node.kubernetes.io/node-name" ) diff --git a/pkg/apis/nfd/v1alpha1/register.go b/pkg/apis/nfd/v1alpha1/register.go index c87a4852a4..8c1d41f5b6 100644 --- a/pkg/apis/nfd/v1alpha1/register.go +++ b/pkg/apis/nfd/v1alpha1/register.go @@ -40,6 +40,7 @@ func Resource(resource string) schema.GroupResource { func addKnownTypes(scheme *runtime.Scheme) error { scheme.AddKnownTypes(SchemeGroupVersion, + &NodeFeature{}, &NodeFeatureRule{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) diff --git a/pkg/apis/nfd/v1alpha1/types.go b/pkg/apis/nfd/v1alpha1/types.go index c057cddf7e..3565db1cd0 100644 --- a/pkg/apis/nfd/v1alpha1/types.go +++ b/pkg/apis/nfd/v1alpha1/types.go @@ -21,6 +21,37 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// NodeFeatureList contains a list of NodeFeature objects. +// +kubebuilder:object:root=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type NodeFeatureList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + + Items []NodeFeature `json:"items"` +} + +// NodeFeature resource holds the features discovered for one node in the +// cluster. +// +kubebuilder:object:root=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +genclient +type NodeFeature struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec NodeFeatureSpec `json:"spec"` +} + +// NodeFeatureSpec describes a NodeFeature object. +type NodeFeatureSpec struct { + // Features is the full "raw" features data that has been discovered. + Features Features `json:"features"` + // Labels is the set of node labels that are requested to be created. + // +optional + Labels map[string]string `json:"labels"` +} + // Features is the collection of all discovered features. // // +protobuf=true From d1c91e129a3381a49eec50efd7f6bde72446e927 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Mon, 4 Jul 2022 18:03:19 +0300 Subject: [PATCH 43/87] apis/nfd: update auto-generated code --- ...featurerule-crd.yaml => nfd-api-crds.yaml} | 111 +++++++++++ ...featurerule-crd.yaml => nfd-api-crds.yaml} | 111 +++++++++++ .../nfd/v1alpha1/zz_generated.deepcopy.go | 81 ++++++++ .../nfd/v1alpha1/fake/fake_nfd_client.go | 4 + .../nfd/v1alpha1/fake/fake_nodefeature.go | 130 +++++++++++++ .../typed/nfd/v1alpha1/generated_expansion.go | 2 + .../typed/nfd/v1alpha1/nfd_client.go | 5 + .../typed/nfd/v1alpha1/nodefeature.go | 178 ++++++++++++++++++ .../informers/externalversions/generic.go | 2 + .../nfd/v1alpha1/interface.go | 7 + .../nfd/v1alpha1/nodefeature.go | 90 +++++++++ .../nfd/v1alpha1/expansion_generated.go | 8 + .../listers/nfd/v1alpha1/nodefeature.go | 99 ++++++++++ 13 files changed, 828 insertions(+) rename deployment/base/nfd-crds/{nodefeaturerule-crd.yaml => nfd-api-crds.yaml} (74%) rename deployment/helm/node-feature-discovery/crds/{nodefeaturerule-crd.yaml => nfd-api-crds.yaml} (74%) create mode 100644 pkg/generated/clientset/versioned/typed/nfd/v1alpha1/fake/fake_nodefeature.go create mode 100644 pkg/generated/clientset/versioned/typed/nfd/v1alpha1/nodefeature.go create mode 100644 pkg/generated/informers/externalversions/nfd/v1alpha1/nodefeature.go create mode 100644 pkg/generated/listers/nfd/v1alpha1/nodefeature.go diff --git a/deployment/base/nfd-crds/nodefeaturerule-crd.yaml b/deployment/base/nfd-crds/nfd-api-crds.yaml similarity index 74% rename from deployment/base/nfd-crds/nodefeaturerule-crd.yaml rename to deployment/base/nfd-crds/nfd-api-crds.yaml index 9a1fc53c83..ab10f15a9c 100644 --- a/deployment/base/nfd-crds/nodefeaturerule-crd.yaml +++ b/deployment/base/nfd-crds/nfd-api-crds.yaml @@ -1,6 +1,117 @@ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: nodefeatures.nfd.k8s-sigs.io +spec: + group: nfd.k8s-sigs.io + names: + kind: NodeFeature + listKind: NodeFeatureList + plural: nodefeatures + singular: nodefeature + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: NodeFeature resource holds the features discovered for one node + in the cluster. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NodeFeatureSpec describes a NodeFeature object. + properties: + features: + description: Features is the full "raw" features data that has been + discovered. + properties: + attributes: + additionalProperties: + description: AttributeFeatureSet is a set of features having + string value. + properties: + elements: + additionalProperties: + type: string + type: object + required: + - elements + type: object + type: object + flags: + additionalProperties: + description: FlagFeatureSet is a set of simple features only + containing names without values. + properties: + elements: + additionalProperties: + description: Nil is a dummy empty struct for protobuf + compatibility + type: object + type: object + required: + - elements + type: object + type: object + instances: + additionalProperties: + description: InstanceFeatureSet is a set of features each of + which is an instance having multiple attributes. + properties: + elements: + items: + description: InstanceFeature represents one instance of + a complex features, e.g. a device. + properties: + attributes: + additionalProperties: + type: string + type: object + required: + - attributes + type: object + type: array + required: + - elements + type: object + type: object + required: + - attributes + - flags + - instances + type: object + labels: + additionalProperties: + type: string + description: Labels is the set of node labels that are requested to + be created. + type: object + required: + - features + type: object + required: + - spec + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.9.2 diff --git a/deployment/helm/node-feature-discovery/crds/nodefeaturerule-crd.yaml b/deployment/helm/node-feature-discovery/crds/nfd-api-crds.yaml similarity index 74% rename from deployment/helm/node-feature-discovery/crds/nodefeaturerule-crd.yaml rename to deployment/helm/node-feature-discovery/crds/nfd-api-crds.yaml index 9a1fc53c83..ab10f15a9c 100644 --- a/deployment/helm/node-feature-discovery/crds/nodefeaturerule-crd.yaml +++ b/deployment/helm/node-feature-discovery/crds/nfd-api-crds.yaml @@ -1,6 +1,117 @@ --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: nodefeatures.nfd.k8s-sigs.io +spec: + group: nfd.k8s-sigs.io + names: + kind: NodeFeature + listKind: NodeFeatureList + plural: nodefeatures + singular: nodefeature + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: NodeFeature resource holds the features discovered for one node + in the cluster. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NodeFeatureSpec describes a NodeFeature object. + properties: + features: + description: Features is the full "raw" features data that has been + discovered. + properties: + attributes: + additionalProperties: + description: AttributeFeatureSet is a set of features having + string value. + properties: + elements: + additionalProperties: + type: string + type: object + required: + - elements + type: object + type: object + flags: + additionalProperties: + description: FlagFeatureSet is a set of simple features only + containing names without values. + properties: + elements: + additionalProperties: + description: Nil is a dummy empty struct for protobuf + compatibility + type: object + type: object + required: + - elements + type: object + type: object + instances: + additionalProperties: + description: InstanceFeatureSet is a set of features each of + which is an instance having multiple attributes. + properties: + elements: + items: + description: InstanceFeature represents one instance of + a complex features, e.g. a device. + properties: + attributes: + additionalProperties: + type: string + type: object + required: + - attributes + type: object + type: array + required: + - elements + type: object + type: object + required: + - attributes + - flags + - instances + type: object + labels: + additionalProperties: + type: string + description: Labels is the set of node labels that are requested to + be created. + type: object + required: + - features + type: object + required: + - spec + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.9.2 diff --git a/pkg/apis/nfd/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/nfd/v1alpha1/zz_generated.deepcopy.go index 47513f1335..07073afac1 100644 --- a/pkg/apis/nfd/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/nfd/v1alpha1/zz_generated.deepcopy.go @@ -342,6 +342,64 @@ func (in *Nil) DeepCopy() *Nil { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeFeature) DeepCopyInto(out *NodeFeature) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeFeature. +func (in *NodeFeature) DeepCopy() *NodeFeature { + if in == nil { + return nil + } + out := new(NodeFeature) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NodeFeature) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeFeatureList) DeepCopyInto(out *NodeFeatureList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NodeFeature, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeFeatureList. +func (in *NodeFeatureList) DeepCopy() *NodeFeatureList { + if in == nil { + return nil + } + out := new(NodeFeatureList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NodeFeatureList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodeFeatureRule) DeepCopyInto(out *NodeFeatureRule) { *out = *in @@ -422,6 +480,29 @@ func (in *NodeFeatureRuleSpec) DeepCopy() *NodeFeatureRuleSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeFeatureSpec) DeepCopyInto(out *NodeFeatureSpec) { + *out = *in + in.Features.DeepCopyInto(&out.Features) + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeFeatureSpec. +func (in *NodeFeatureSpec) DeepCopy() *NodeFeatureSpec { + if in == nil { + return nil + } + out := new(NodeFeatureSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Rule) DeepCopyInto(out *Rule) { *out = *in diff --git a/pkg/generated/clientset/versioned/typed/nfd/v1alpha1/fake/fake_nfd_client.go b/pkg/generated/clientset/versioned/typed/nfd/v1alpha1/fake/fake_nfd_client.go index 7133268330..4022cd3691 100644 --- a/pkg/generated/clientset/versioned/typed/nfd/v1alpha1/fake/fake_nfd_client.go +++ b/pkg/generated/clientset/versioned/typed/nfd/v1alpha1/fake/fake_nfd_client.go @@ -28,6 +28,10 @@ type FakeNfdV1alpha1 struct { *testing.Fake } +func (c *FakeNfdV1alpha1) NodeFeatures(namespace string) v1alpha1.NodeFeatureInterface { + return &FakeNodeFeatures{c, namespace} +} + func (c *FakeNfdV1alpha1) NodeFeatureRules() v1alpha1.NodeFeatureRuleInterface { return &FakeNodeFeatureRules{c} } diff --git a/pkg/generated/clientset/versioned/typed/nfd/v1alpha1/fake/fake_nodefeature.go b/pkg/generated/clientset/versioned/typed/nfd/v1alpha1/fake/fake_nodefeature.go new file mode 100644 index 0000000000..1986bf6181 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/nfd/v1alpha1/fake/fake_nodefeature.go @@ -0,0 +1,130 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" + v1alpha1 "sigs.k8s.io/node-feature-discovery/pkg/apis/nfd/v1alpha1" +) + +// FakeNodeFeatures implements NodeFeatureInterface +type FakeNodeFeatures struct { + Fake *FakeNfdV1alpha1 + ns string +} + +var nodefeaturesResource = schema.GroupVersionResource{Group: "nfd.k8s-sigs.io", Version: "v1alpha1", Resource: "nodefeatures"} + +var nodefeaturesKind = schema.GroupVersionKind{Group: "nfd.k8s-sigs.io", Version: "v1alpha1", Kind: "NodeFeature"} + +// Get takes name of the nodeFeature, and returns the corresponding nodeFeature object, and an error if there is any. +func (c *FakeNodeFeatures) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.NodeFeature, err error) { + obj, err := c.Fake. + Invokes(testing.NewGetAction(nodefeaturesResource, c.ns, name), &v1alpha1.NodeFeature{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.NodeFeature), err +} + +// List takes label and field selectors, and returns the list of NodeFeatures that match those selectors. +func (c *FakeNodeFeatures) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.NodeFeatureList, err error) { + obj, err := c.Fake. + Invokes(testing.NewListAction(nodefeaturesResource, nodefeaturesKind, c.ns, opts), &v1alpha1.NodeFeatureList{}) + + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.NodeFeatureList{ListMeta: obj.(*v1alpha1.NodeFeatureList).ListMeta} + for _, item := range obj.(*v1alpha1.NodeFeatureList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested nodeFeatures. +func (c *FakeNodeFeatures) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchAction(nodefeaturesResource, c.ns, opts)) + +} + +// Create takes the representation of a nodeFeature and creates it. Returns the server's representation of the nodeFeature, and an error, if there is any. +func (c *FakeNodeFeatures) Create(ctx context.Context, nodeFeature *v1alpha1.NodeFeature, opts v1.CreateOptions) (result *v1alpha1.NodeFeature, err error) { + obj, err := c.Fake. + Invokes(testing.NewCreateAction(nodefeaturesResource, c.ns, nodeFeature), &v1alpha1.NodeFeature{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.NodeFeature), err +} + +// Update takes the representation of a nodeFeature and updates it. Returns the server's representation of the nodeFeature, and an error, if there is any. +func (c *FakeNodeFeatures) Update(ctx context.Context, nodeFeature *v1alpha1.NodeFeature, opts v1.UpdateOptions) (result *v1alpha1.NodeFeature, err error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateAction(nodefeaturesResource, c.ns, nodeFeature), &v1alpha1.NodeFeature{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.NodeFeature), err +} + +// Delete takes name of the nodeFeature and deletes it. Returns an error if one occurs. +func (c *FakeNodeFeatures) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteActionWithOptions(nodefeaturesResource, c.ns, name, opts), &v1alpha1.NodeFeature{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeNodeFeatures) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionAction(nodefeaturesResource, c.ns, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha1.NodeFeatureList{}) + return err +} + +// Patch applies the patch and returns the patched nodeFeature. +func (c *FakeNodeFeatures) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.NodeFeature, err error) { + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(nodefeaturesResource, c.ns, name, pt, data, subresources...), &v1alpha1.NodeFeature{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.NodeFeature), err +} diff --git a/pkg/generated/clientset/versioned/typed/nfd/v1alpha1/generated_expansion.go b/pkg/generated/clientset/versioned/typed/nfd/v1alpha1/generated_expansion.go index 8807cd64dd..cf392dd100 100644 --- a/pkg/generated/clientset/versioned/typed/nfd/v1alpha1/generated_expansion.go +++ b/pkg/generated/clientset/versioned/typed/nfd/v1alpha1/generated_expansion.go @@ -18,4 +18,6 @@ limitations under the License. package v1alpha1 +type NodeFeatureExpansion interface{} + type NodeFeatureRuleExpansion interface{} diff --git a/pkg/generated/clientset/versioned/typed/nfd/v1alpha1/nfd_client.go b/pkg/generated/clientset/versioned/typed/nfd/v1alpha1/nfd_client.go index 034d5cbe74..8ecaf1998d 100644 --- a/pkg/generated/clientset/versioned/typed/nfd/v1alpha1/nfd_client.go +++ b/pkg/generated/clientset/versioned/typed/nfd/v1alpha1/nfd_client.go @@ -28,6 +28,7 @@ import ( type NfdV1alpha1Interface interface { RESTClient() rest.Interface + NodeFeaturesGetter NodeFeatureRulesGetter } @@ -36,6 +37,10 @@ type NfdV1alpha1Client struct { restClient rest.Interface } +func (c *NfdV1alpha1Client) NodeFeatures(namespace string) NodeFeatureInterface { + return newNodeFeatures(c, namespace) +} + func (c *NfdV1alpha1Client) NodeFeatureRules() NodeFeatureRuleInterface { return newNodeFeatureRules(c) } diff --git a/pkg/generated/clientset/versioned/typed/nfd/v1alpha1/nodefeature.go b/pkg/generated/clientset/versioned/typed/nfd/v1alpha1/nodefeature.go new file mode 100644 index 0000000000..bf6d9b1759 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/nfd/v1alpha1/nodefeature.go @@ -0,0 +1,178 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + "time" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" + v1alpha1 "sigs.k8s.io/node-feature-discovery/pkg/apis/nfd/v1alpha1" + scheme "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned/scheme" +) + +// NodeFeaturesGetter has a method to return a NodeFeatureInterface. +// A group's client should implement this interface. +type NodeFeaturesGetter interface { + NodeFeatures(namespace string) NodeFeatureInterface +} + +// NodeFeatureInterface has methods to work with NodeFeature resources. +type NodeFeatureInterface interface { + Create(ctx context.Context, nodeFeature *v1alpha1.NodeFeature, opts v1.CreateOptions) (*v1alpha1.NodeFeature, error) + Update(ctx context.Context, nodeFeature *v1alpha1.NodeFeature, opts v1.UpdateOptions) (*v1alpha1.NodeFeature, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.NodeFeature, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.NodeFeatureList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.NodeFeature, err error) + NodeFeatureExpansion +} + +// nodeFeatures implements NodeFeatureInterface +type nodeFeatures struct { + client rest.Interface + ns string +} + +// newNodeFeatures returns a NodeFeatures +func newNodeFeatures(c *NfdV1alpha1Client, namespace string) *nodeFeatures { + return &nodeFeatures{ + client: c.RESTClient(), + ns: namespace, + } +} + +// Get takes name of the nodeFeature, and returns the corresponding nodeFeature object, and an error if there is any. +func (c *nodeFeatures) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.NodeFeature, err error) { + result = &v1alpha1.NodeFeature{} + err = c.client.Get(). + Namespace(c.ns). + Resource("nodefeatures"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of NodeFeatures that match those selectors. +func (c *nodeFeatures) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.NodeFeatureList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1alpha1.NodeFeatureList{} + err = c.client.Get(). + Namespace(c.ns). + Resource("nodefeatures"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested nodeFeatures. +func (c *nodeFeatures) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Namespace(c.ns). + Resource("nodefeatures"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a nodeFeature and creates it. Returns the server's representation of the nodeFeature, and an error, if there is any. +func (c *nodeFeatures) Create(ctx context.Context, nodeFeature *v1alpha1.NodeFeature, opts v1.CreateOptions) (result *v1alpha1.NodeFeature, err error) { + result = &v1alpha1.NodeFeature{} + err = c.client.Post(). + Namespace(c.ns). + Resource("nodefeatures"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(nodeFeature). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a nodeFeature and updates it. Returns the server's representation of the nodeFeature, and an error, if there is any. +func (c *nodeFeatures) Update(ctx context.Context, nodeFeature *v1alpha1.NodeFeature, opts v1.UpdateOptions) (result *v1alpha1.NodeFeature, err error) { + result = &v1alpha1.NodeFeature{} + err = c.client.Put(). + Namespace(c.ns). + Resource("nodefeatures"). + Name(nodeFeature.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(nodeFeature). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the nodeFeature and deletes it. Returns an error if one occurs. +func (c *nodeFeatures) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("nodefeatures"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *nodeFeatures) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Namespace(c.ns). + Resource("nodefeatures"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched nodeFeature. +func (c *nodeFeatures) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.NodeFeature, err error) { + result = &v1alpha1.NodeFeature{} + err = c.client.Patch(pt). + Namespace(c.ns). + Resource("nodefeatures"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/pkg/generated/informers/externalversions/generic.go b/pkg/generated/informers/externalversions/generic.go index 453de7f151..5c2b9eda3b 100644 --- a/pkg/generated/informers/externalversions/generic.go +++ b/pkg/generated/informers/externalversions/generic.go @@ -53,6 +53,8 @@ func (f *genericInformer) Lister() cache.GenericLister { func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) { switch resource { // Group=nfd.k8s-sigs.io, Version=v1alpha1 + case v1alpha1.SchemeGroupVersion.WithResource("nodefeatures"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Nfd().V1alpha1().NodeFeatures().Informer()}, nil case v1alpha1.SchemeGroupVersion.WithResource("nodefeaturerules"): return &genericInformer{resource: resource.GroupResource(), informer: f.Nfd().V1alpha1().NodeFeatureRules().Informer()}, nil diff --git a/pkg/generated/informers/externalversions/nfd/v1alpha1/interface.go b/pkg/generated/informers/externalversions/nfd/v1alpha1/interface.go index 5f476d8873..1cae0cfc24 100644 --- a/pkg/generated/informers/externalversions/nfd/v1alpha1/interface.go +++ b/pkg/generated/informers/externalversions/nfd/v1alpha1/interface.go @@ -24,6 +24,8 @@ import ( // Interface provides access to all the informers in this group version. type Interface interface { + // NodeFeatures returns a NodeFeatureInformer. + NodeFeatures() NodeFeatureInformer // NodeFeatureRules returns a NodeFeatureRuleInformer. NodeFeatureRules() NodeFeatureRuleInformer } @@ -39,6 +41,11 @@ func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakList return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} } +// NodeFeatures returns a NodeFeatureInformer. +func (v *version) NodeFeatures() NodeFeatureInformer { + return &nodeFeatureInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} + // NodeFeatureRules returns a NodeFeatureRuleInformer. func (v *version) NodeFeatureRules() NodeFeatureRuleInformer { return &nodeFeatureRuleInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} diff --git a/pkg/generated/informers/externalversions/nfd/v1alpha1/nodefeature.go b/pkg/generated/informers/externalversions/nfd/v1alpha1/nodefeature.go new file mode 100644 index 0000000000..20ea65f561 --- /dev/null +++ b/pkg/generated/informers/externalversions/nfd/v1alpha1/nodefeature.go @@ -0,0 +1,90 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + time "time" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" + nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/pkg/apis/nfd/v1alpha1" + versioned "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned" + internalinterfaces "sigs.k8s.io/node-feature-discovery/pkg/generated/informers/externalversions/internalinterfaces" + v1alpha1 "sigs.k8s.io/node-feature-discovery/pkg/generated/listers/nfd/v1alpha1" +) + +// NodeFeatureInformer provides access to a shared informer and lister for +// NodeFeatures. +type NodeFeatureInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1alpha1.NodeFeatureLister +} + +type nodeFeatureInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewNodeFeatureInformer constructs a new informer for NodeFeature type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewNodeFeatureInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredNodeFeatureInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredNodeFeatureInformer constructs a new informer for NodeFeature type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredNodeFeatureInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.NfdV1alpha1().NodeFeatures(namespace).List(context.TODO(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.NfdV1alpha1().NodeFeatures(namespace).Watch(context.TODO(), options) + }, + }, + &nfdv1alpha1.NodeFeature{}, + resyncPeriod, + indexers, + ) +} + +func (f *nodeFeatureInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredNodeFeatureInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *nodeFeatureInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&nfdv1alpha1.NodeFeature{}, f.defaultInformer) +} + +func (f *nodeFeatureInformer) Lister() v1alpha1.NodeFeatureLister { + return v1alpha1.NewNodeFeatureLister(f.Informer().GetIndexer()) +} diff --git a/pkg/generated/listers/nfd/v1alpha1/expansion_generated.go b/pkg/generated/listers/nfd/v1alpha1/expansion_generated.go index a072a4f4ae..5e6fc01e4d 100644 --- a/pkg/generated/listers/nfd/v1alpha1/expansion_generated.go +++ b/pkg/generated/listers/nfd/v1alpha1/expansion_generated.go @@ -18,6 +18,14 @@ limitations under the License. package v1alpha1 +// NodeFeatureListerExpansion allows custom methods to be added to +// NodeFeatureLister. +type NodeFeatureListerExpansion interface{} + +// NodeFeatureNamespaceListerExpansion allows custom methods to be added to +// NodeFeatureNamespaceLister. +type NodeFeatureNamespaceListerExpansion interface{} + // NodeFeatureRuleListerExpansion allows custom methods to be added to // NodeFeatureRuleLister. type NodeFeatureRuleListerExpansion interface{} diff --git a/pkg/generated/listers/nfd/v1alpha1/nodefeature.go b/pkg/generated/listers/nfd/v1alpha1/nodefeature.go new file mode 100644 index 0000000000..011dffd559 --- /dev/null +++ b/pkg/generated/listers/nfd/v1alpha1/nodefeature.go @@ -0,0 +1,99 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" + v1alpha1 "sigs.k8s.io/node-feature-discovery/pkg/apis/nfd/v1alpha1" +) + +// NodeFeatureLister helps list NodeFeatures. +// All objects returned here must be treated as read-only. +type NodeFeatureLister interface { + // List lists all NodeFeatures in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1alpha1.NodeFeature, err error) + // NodeFeatures returns an object that can list and get NodeFeatures. + NodeFeatures(namespace string) NodeFeatureNamespaceLister + NodeFeatureListerExpansion +} + +// nodeFeatureLister implements the NodeFeatureLister interface. +type nodeFeatureLister struct { + indexer cache.Indexer +} + +// NewNodeFeatureLister returns a new NodeFeatureLister. +func NewNodeFeatureLister(indexer cache.Indexer) NodeFeatureLister { + return &nodeFeatureLister{indexer: indexer} +} + +// List lists all NodeFeatures in the indexer. +func (s *nodeFeatureLister) List(selector labels.Selector) (ret []*v1alpha1.NodeFeature, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.NodeFeature)) + }) + return ret, err +} + +// NodeFeatures returns an object that can list and get NodeFeatures. +func (s *nodeFeatureLister) NodeFeatures(namespace string) NodeFeatureNamespaceLister { + return nodeFeatureNamespaceLister{indexer: s.indexer, namespace: namespace} +} + +// NodeFeatureNamespaceLister helps list and get NodeFeatures. +// All objects returned here must be treated as read-only. +type NodeFeatureNamespaceLister interface { + // List lists all NodeFeatures in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1alpha1.NodeFeature, err error) + // Get retrieves the NodeFeature from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*v1alpha1.NodeFeature, error) + NodeFeatureNamespaceListerExpansion +} + +// nodeFeatureNamespaceLister implements the NodeFeatureNamespaceLister +// interface. +type nodeFeatureNamespaceLister struct { + indexer cache.Indexer + namespace string +} + +// List lists all NodeFeatures in the indexer for a given namespace. +func (s nodeFeatureNamespaceLister) List(selector labels.Selector) (ret []*v1alpha1.NodeFeature, err error) { + err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.NodeFeature)) + }) + return ret, err +} + +// Get retrieves the NodeFeature from the indexer for a given namespace and name. +func (s nodeFeatureNamespaceLister) Get(name string) (*v1alpha1.NodeFeature, error) { + obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v1alpha1.Resource("nodefeature"), name) + } + return obj.(*v1alpha1.NodeFeature), nil +} From 237494463b4b12e6a28d7207b21ccc0350dbc327 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Fri, 12 Aug 2022 13:10:48 +0300 Subject: [PATCH 44/87] nfd-worker: support creating NodeFeatures object Support the new NodeFeatures object of the NFD CRD api. Add two new command line options to nfd-worker: -kubeconfig specifies the kubeconfig to use for connecting k8s api (defaults to empty which implies in-cluster config) -enable-nodefeature-api enable the NodeFeature CRD API for communicating node features to nfd-master, will also automatically disable gRPC (defgault to false) No config file option for selecting the API is available as there should be no need for dynamically selecting between gRPC and CRD. The nfd-master configuration must be changed in tandem and it is safer (and avoid awkward configuration races) to configure the whole NFD deployment at once. Default behavior of nfd-worker is not changed i.e. NodeFeatures object creation is not enabled by default (but must be enabled with the command line flag). The patch also updates the kustomize and Helm deployment, adding RBAC rules for nfd-worker and updating the example worker configuration. --- cmd/nfd-worker/main.go | 6 +- deployment/base/rbac/kustomization.yaml | 3 + deployment/base/rbac/worker-role.yaml | 13 ++ deployment/base/rbac/worker-rolebinding.yaml | 12 ++ .../base/rbac/worker-serviceaccount.yaml | 4 + .../worker-daemonset/worker-daemonset.yaml | 1 + deployment/base/worker-job/worker-job.yaml | 1 + .../templates/role.yaml | 18 +++ .../templates/rolebinding.yaml | 17 +++ .../templates/worker.yaml | 3 + .../helm/node-feature-discovery/values.yaml | 5 + docs/deployment/helm.md | 2 + .../reference/worker-commandline-reference.md | 44 ++++++- .../worker-configuration-reference.md | 12 +- pkg/nfd-client/base.go | 1 + pkg/nfd-client/worker/nfd-worker.go | 117 ++++++++++++++++-- 16 files changed, 244 insertions(+), 15 deletions(-) create mode 100644 deployment/base/rbac/worker-role.yaml create mode 100644 deployment/base/rbac/worker-rolebinding.yaml create mode 100644 deployment/base/rbac/worker-serviceaccount.yaml create mode 100644 deployment/helm/node-feature-discovery/templates/role.yaml create mode 100644 deployment/helm/node-feature-discovery/templates/rolebinding.yaml diff --git a/cmd/nfd-worker/main.go b/cmd/nfd-worker/main.go index 90892380f1..30e3290d17 100644 --- a/cmd/nfd-worker/main.go +++ b/cmd/nfd-worker/main.go @@ -101,6 +101,10 @@ func initFlags(flagset *flag.FlagSet) (*worker.Args, *worker.ConfigOverrideArgs) "Config file to use.") flagset.StringVar(&args.KeyFile, "key-file", "", "Private key matching -cert-file") + flagset.BoolVar(&args.EnableNodeFeatureApi, "enable-nodefeature-api", false, + "Enable the NodeFeature CRD API for communicating with nfd-master. This will automatically disable the gRPC communication.") + flagset.StringVar(&args.Kubeconfig, "kubeconfig", "", + "Kubeconfig to use") flagset.BoolVar(&args.Oneshot, "oneshot", false, "Do not publish feature labels") flagset.StringVar(&args.Options, "options", "", @@ -119,7 +123,7 @@ func initFlags(flagset *flag.FlagSet) (*worker.Args, *worker.ConfigOverrideArgs) LabelSources: &utils.StringSliceVal{}, } overrides.NoPublish = flagset.Bool("no-publish", false, - "Do not publish discovered features, disable connection to nfd-master.") + "Do not publish discovered features, disable connection to nfd-master and don't create NodeFeature object.") flagset.Var(overrides.FeatureSources, "feature-sources", "Comma separated list of feature sources. Special value 'all' enables all sources. "+ "Prefix the source name with '-' to disable it.") diff --git a/deployment/base/rbac/kustomization.yaml b/deployment/base/rbac/kustomization.yaml index fdadb5675e..6eb2d8a8c7 100644 --- a/deployment/base/rbac/kustomization.yaml +++ b/deployment/base/rbac/kustomization.yaml @@ -7,3 +7,6 @@ resources: - master-serviceaccount.yaml - master-clusterrole.yaml - master-clusterrolebinding.yaml +- worker-serviceaccount.yaml +- worker-role.yaml +- worker-rolebinding.yaml diff --git a/deployment/base/rbac/worker-role.yaml b/deployment/base/rbac/worker-role.yaml new file mode 100644 index 0000000000..72f261e9e7 --- /dev/null +++ b/deployment/base/rbac/worker-role.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: nfd-worker +rules: +- apiGroups: + - nfd.k8s-sigs.io + resources: + - nodefeatures + verbs: + - create + - get + - update diff --git a/deployment/base/rbac/worker-rolebinding.yaml b/deployment/base/rbac/worker-rolebinding.yaml new file mode 100644 index 0000000000..707c75fbb6 --- /dev/null +++ b/deployment/base/rbac/worker-rolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: nfd-worker +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nfd-worker +subjects: +- kind: ServiceAccount + name: nfd-worker + namespace: default diff --git a/deployment/base/rbac/worker-serviceaccount.yaml b/deployment/base/rbac/worker-serviceaccount.yaml new file mode 100644 index 0000000000..442b06c5e2 --- /dev/null +++ b/deployment/base/rbac/worker-serviceaccount.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nfd-worker diff --git a/deployment/base/worker-daemonset/worker-daemonset.yaml b/deployment/base/worker-daemonset/worker-daemonset.yaml index 8f02fd0e13..f93bf175f7 100644 --- a/deployment/base/worker-daemonset/worker-daemonset.yaml +++ b/deployment/base/worker-daemonset/worker-daemonset.yaml @@ -13,6 +13,7 @@ spec: labels: app: nfd-worker spec: + serviceAccount: nfd-worker dnsPolicy: ClusterFirstWithHostNet containers: - name: nfd-worker diff --git a/deployment/base/worker-job/worker-job.yaml b/deployment/base/worker-job/worker-job.yaml index dd2f671dc4..6cf2c9dbf2 100644 --- a/deployment/base/worker-job/worker-job.yaml +++ b/deployment/base/worker-job/worker-job.yaml @@ -12,6 +12,7 @@ spec: labels: app: nfd-worker spec: + serviceAccount: nfd-worker dnsPolicy: ClusterFirstWithHostNet restartPolicy: Never affinity: diff --git a/deployment/helm/node-feature-discovery/templates/role.yaml b/deployment/helm/node-feature-discovery/templates/role.yaml new file mode 100644 index 0000000000..f63cb8ff4f --- /dev/null +++ b/deployment/helm/node-feature-discovery/templates/role.yaml @@ -0,0 +1,18 @@ +{{- if .Values.worker.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-worker + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +rules: +- apiGroups: + - nfd.k8s-sigs.io + resources: + - nodefeatures + verbs: + - create + - get + - update +{{- end }} + diff --git a/deployment/helm/node-feature-discovery/templates/rolebinding.yaml b/deployment/helm/node-feature-discovery/templates/rolebinding.yaml new file mode 100644 index 0000000000..30a00381f0 --- /dev/null +++ b/deployment/helm/node-feature-discovery/templates/rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if .Values.worker.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-worker + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "node-feature-discovery.fullname" . }}-worker +subjects: +- kind: ServiceAccount + name: {{ include "node-feature-discovery.worker.serviceAccountName" . }} + namespace: {{ include "node-feature-discovery.namespace" . }} +{{- end }} + diff --git a/deployment/helm/node-feature-discovery/templates/worker.yaml b/deployment/helm/node-feature-discovery/templates/worker.yaml index f48aade328..e723cc5cbe 100644 --- a/deployment/helm/node-feature-discovery/templates/worker.yaml +++ b/deployment/helm/node-feature-discovery/templates/worker.yaml @@ -46,6 +46,9 @@ spec: - "nfd-worker" args: - "--server={{ include "node-feature-discovery.fullname" . }}-master:{{ .Values.master.service.port }}" + {{- if .Values.enableNodeFeatureApi }} + - "-enable-nodefeature-api" + {{- end }} {{- if .Values.tls.enable }} - "--ca-file=/etc/kubernetes/node-feature-discovery/certs/ca.crt" - "--key-file=/etc/kubernetes/node-feature-discovery/certs/tls.key" diff --git a/deployment/helm/node-feature-discovery/values.yaml b/deployment/helm/node-feature-discovery/values.yaml index 2b3a5942aa..89df864381 100644 --- a/deployment/helm/node-feature-discovery/values.yaml +++ b/deployment/helm/node-feature-discovery/values.yaml @@ -10,6 +10,8 @@ nameOverride: "" fullnameOverride: "" namespaceOverride: "" +enableNodeFeatureApi: false + master: instance: extraLabelNs: [] @@ -338,6 +340,9 @@ worker: # If not set and create is true, a name is generated using the fullname template name: + rbac: + create: true + # Allow users to mount the hostPath /usr/src, useful for RHCOS on s390x # Does not work on systems without /usr/src AND a read-only /usr, such as Talos mountUsrSrc: false diff --git a/docs/deployment/helm.md b/docs/deployment/helm.md index e2f0435cd3..6e032026e1 100644 --- a/docs/deployment/helm.md +++ b/docs/deployment/helm.md @@ -97,6 +97,7 @@ We have introduced the following Chart parameters. | `fullnameOverride` | string | | Override a default fully qualified app name | | `tls.enable` | bool | false | Specifies whether to use TLS for communications between components | | `tls.certManager` | bool | false | If enabled, requires [cert-manager](https://cert-manager.io/docs/) to be installed and will automatically create the required TLS certificates | +| `enableNodeFeatureApi` | bool | false | Enable the NodeFeature CRD API for communicating node features. This will automatically disable the gRPC communication. ### Master pod parameters @@ -134,6 +135,7 @@ We have introduced the following Chart parameters. | `worker.serviceAccount.create` | bool | true | Specifies whether a service account for nfd-worker should be created | `worker.serviceAccount.annotations` | dict | {} | Annotations to add to the service account for nfd-worker | `worker.serviceAccount.name` | string | | The name of the service account to use for nfd-worker. If not set and create is true, a name is generated using the fullname template (suffixed with `-worker`) +| `worker.rbac.create` | bool | true | Specifies whether to create [RBAC][rbac] configuration for nfd-worker | `worker.mountUsrSrc` | bool | false | Specifies whether to allow users to mount the hostpath /user/src. Does not work on systems without /usr/src AND a read-only /usr | | `worker.resources` | dict | {} | NFD worker pod [resources management](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) | | `worker.nodeSelector` | dict | {} | NFD worker pod [node selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) | diff --git a/docs/reference/worker-commandline-reference.md b/docs/reference/worker-commandline-reference.md index 886faf4c7e..cf6651b8f2 100644 --- a/docs/reference/worker-commandline-reference.md +++ b/docs/reference/worker-commandline-reference.md @@ -122,6 +122,22 @@ Example: nfd-worker -key-file=/opt/nfd/worker.key -cert-file=/opt/nfd/worker.crt -ca-file=/opt/nfd/ca.crt ``` +### -kubeconfig + +The `-kubeconfig` flag specifies the kubeconfig to use for connecting to the +Kubernetes API server. It is only needed for manipulating NodeFeature +objects, and thus the flag only takes effect when +[`-enable-nodefeature-api`](#-enable-nodefeature-api)) is specified. An empty +value (which is also the default) implies in-cluster kubeconfig. + +Default: *empty* + +Example: + +```bash +nfd-worker -kubeconfig ${HOME}/.kube/config +``` + ### -server-name-override The `-server-name-override` flag specifies the common name (CN) which to @@ -178,11 +194,33 @@ Example: nfd-worker -label-sources=kernel,system,local ``` +### -enable-nodefeature-api + +The `-enable-nodefeature-api` flag enables the experimental NodeFeature CRD API +for communicating with nfd-master. This will also automatically disable the +gRPC communication to nfd-master. When enabled, nfd-worker will create per-node +NodeFeature objects the contain all discovered node features and the set of +feature labels to be created. + +Default: false + +Example: + +```bash +nfd-worker -enable-nodefeature-api +``` + ### -no-publish -The `-no-publish` flag disables all communication with the nfd-master, making -it a "dry-run" flag for nfd-worker. NFD-Worker runs feature detection normally, -but no labeling requests are sent to nfd-master. +The `-no-publish` flag disables all communication with the nfd-master and the +Kubernetes API server. It is effectively a "dry-run" flag for nfd-worker. +NFD-Worker runs feature detection normally, but no labeling requests are sent +to nfd-master and no NodeFeature objects are created or updated in the API +server. + +Note: This flag takes precedence over the +[`core.noPublish`](worker-configuration-reference#corenopublish) +configuration file option. Default: *false* diff --git a/docs/reference/worker-configuration-reference.md b/docs/reference/worker-configuration-reference.md index 5dd239e811..48b13307cb 100644 --- a/docs/reference/worker-configuration-reference.md +++ b/docs/reference/worker-configuration-reference.md @@ -131,10 +131,14 @@ core: ### core.noPublish Setting `core.noPublish` to `true` disables all communication with the -nfd-master. It is effectively a "dry-run" flag: nfd-worker runs feature -detection normally, but no labeling requests are sent to nfd-master. - -Note: Overridden by the `-no-publish` command line flag (if specified). +nfd-master and the Kubernetes API server. It is effectively a "dry-run" option. +NFD-Worker runs feature detection normally, but no labeling requests are sent +to nfd-master and no NodeFeature objects are created or updated in the API +server. + +Note: Overridden by the +[`-no-publish`](worker-commandline-reference#-no-publish) command line flag (if +specified). Default: `false` diff --git a/pkg/nfd-client/base.go b/pkg/nfd-client/base.go index fa5e0b01a2..c965535421 100644 --- a/pkg/nfd-client/base.go +++ b/pkg/nfd-client/base.go @@ -49,6 +49,7 @@ type Args struct { CaFile string CertFile string KeyFile string + Kubeconfig string Server string ServerNameOverride string diff --git a/pkg/nfd-client/worker/nfd-worker.go b/pkg/nfd-client/worker/nfd-worker.go index a7fdeb1f6e..abc0fdb4a8 100644 --- a/pkg/nfd-client/worker/nfd-worker.go +++ b/pkg/nfd-client/worker/nfd-worker.go @@ -27,10 +27,16 @@ import ( "time" "golang.org/x/net/context" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/klog/v2" "sigs.k8s.io/yaml" + apiequality "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/node-feature-discovery/pkg/apihelper" + nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/pkg/apis/nfd/v1alpha1" + nfdclient "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned" pb "sigs.k8s.io/node-feature-discovery/pkg/labeler" clientcommon "sigs.k8s.io/node-feature-discovery/pkg/nfd-client" "sigs.k8s.io/node-feature-discovery/pkg/utils" @@ -76,9 +82,10 @@ type Labels map[string]string type Args struct { clientcommon.Args - ConfigFile string - Oneshot bool - Options string + ConfigFile string + EnableNodeFeatureApi bool + Oneshot bool + Options string Klog map[string]*utils.KlogFlagVal Overrides ConfigOverrideArgs @@ -101,6 +108,7 @@ type nfdWorker struct { config *NFDConfig kubernetesNamespace string grpcClient pb.LabelerClient + nfdClient *nfdclient.Clientset stop chan struct{} // channel for signaling stop featureSources []source.FeatureSource labelSources []source.LabelSource @@ -150,6 +158,7 @@ func newDefaultConfig() *NFDConfig { func (w *nfdWorker) Run() error { klog.Infof("Node Feature Discovery Worker %s", version.Get()) klog.Infof("NodeName: '%s'", clientcommon.NodeName()) + klog.Infof("Kubernetes namespace: '%s'", w.kubernetesNamespace) // Create watcher for config file and read initial configuration configWatch, err := utils.CreateFsWatcher(time.Second, w.configFilePath) @@ -185,9 +194,8 @@ func (w *nfdWorker) Run() error { // Update the node with the feature labels. if !w.config.Core.NoPublish { - err := w.advertiseFeatureLabels(labels) - if err != nil { - return fmt.Errorf("failed to advertise labels: %s", err.Error()) + if err := w.advertiseFeatures(labels); err != nil { + return err } } @@ -205,7 +213,7 @@ func (w *nfdWorker) Run() error { return err } // Manage connection to master - if w.config.Core.NoPublish { + if w.config.Core.NoPublish || !w.args.EnableNodeFeatureApi { w.GrpcDisconnect() } @@ -524,6 +532,22 @@ func getFeatureLabels(source source.LabelSource, labelWhiteList regexp.Regexp) ( return labels, nil } +// advertiseFeatures advertises the features of a Kubernetes node +func (w *nfdWorker) advertiseFeatures(labels Labels) error { + if w.args.EnableNodeFeatureApi { + // Create/update NodeFeature CR object + if err := w.updateNodeFeatureObject(labels); err != nil { + return fmt.Errorf("failed to advertise features (via CRD API): %w", err) + } + } else { + // Create/update feature labels through gRPC connection to nfd-master + if err := w.advertiseFeatureLabels(labels); err != nil { + return fmt.Errorf("failed to advertise features (via gRPC): %w", err) + } + } + return nil +} + // advertiseFeatureLabels advertises the feature labels to a Kubernetes node // via the NFD server. func (w *nfdWorker) advertiseFeatureLabels(labels Labels) error { @@ -551,6 +575,85 @@ func (w *nfdWorker) advertiseFeatureLabels(labels Labels) error { return nil } +// updateNodeFeatureObject creates/updates the node-specific NodeFeature custom resource. +func (m *nfdWorker) updateNodeFeatureObject(labels Labels) error { + cli, err := m.getNfdClient() + if err != nil { + return err + } + nodename := clientcommon.NodeName() + namespace := m.kubernetesNamespace + + features := source.GetAllFeatures() + + // TODO: we could implement some simple caching of the object, only get it + // every 10 minutes or so because nobody else should really be modifying it + if nfr, err := cli.NfdV1alpha1().NodeFeatures(namespace).Get(context.TODO(), nodename, metav1.GetOptions{}); errors.IsNotFound(err) { + klog.Infof("creating NodeFeature object %q", nodename) + nfr = &nfdv1alpha1.NodeFeature{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodename, + Annotations: map[string]string{nfdv1alpha1.WorkerVersionAnnotation: version.Get()}, + Labels: map[string]string{nfdv1alpha1.NodeFeatureObjNodeNameLabel: nodename}, + }, + Spec: nfdv1alpha1.NodeFeatureSpec{ + Features: *features, + Labels: labels, + }, + } + + nfrCreated, err := cli.NfdV1alpha1().NodeFeatures(namespace).Create(context.TODO(), nfr, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create NodeFeature object %q: %w", nfr.Name, err) + } + + utils.KlogDump(4, "NodeFeature object created:", " ", nfrCreated) + } else if err != nil { + return fmt.Errorf("failed to get NodeFeature object: %w", err) + } else { + + nfrUpdated := nfr.DeepCopy() + nfrUpdated.Annotations = map[string]string{nfdv1alpha1.WorkerVersionAnnotation: version.Get()} + nfrUpdated.Labels = map[string]string{nfdv1alpha1.NodeFeatureObjNodeNameLabel: nodename} + nfrUpdated.Spec = nfdv1alpha1.NodeFeatureSpec{ + Features: *features, + Labels: labels, + } + + if !apiequality.Semantic.DeepEqual(nfr, nfrUpdated) { + klog.Infof("updating NodeFeature object %q", nodename) + nfrUpdated, err = cli.NfdV1alpha1().NodeFeatures(namespace).Update(context.TODO(), nfrUpdated, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to update NodeFeature object %q: %w", nfr.Name, err) + } + utils.KlogDump(4, "NodeFeature object updated:", " ", nfrUpdated) + } else { + klog.V(1).Info("no changes in NodeFeature object, not updating") + } + } + return nil +} + +// getNfdClient returns the clientset for using the nfd CRD api +func (m *nfdWorker) getNfdClient() (*nfdclient.Clientset, error) { + if m.nfdClient != nil { + return m.nfdClient, nil + } + + kubeconfig, err := apihelper.GetKubeconfig(m.args.Kubeconfig) + if err != nil { + return nil, err + } + + c, err := nfdclient.NewForConfig(kubeconfig) + if err != nil { + return nil, err + } + + m.nfdClient = c + return c, nil +} + // UnmarshalJSON implements the Unmarshaler interface from "encoding/json" func (d *duration) UnmarshalJSON(data []byte) error { var v interface{} From 6ddd87e465f65bba3a679b8eb6038b57a906d35a Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Wed, 12 Oct 2022 13:45:11 +0300 Subject: [PATCH 45/87] nfd-master: support NodeFeature objects Add initial support for handling NodeFeature objects. With this patch nfd-master watches NodeFeature objects in all namespaces and reacts to changes in any of these. The node which a certain NodeFeature object affects is determined by the "nfd.node.kubernetes.io/node-name" annotation of the object. When a NodeFeature object targeting certain node is changed, nfd-master needs to process all other objects targeting the same node, too, because there may be dependencies between them. Add a new command line flag for selecting between gRPC and NodeFeature CRD API as the source of feature requests. Enabling NodeFeature API disables the gRPC interface. -enable-nodefeature-api enable NodeFeature CRD API for incoming feature requests, will disable the gRPC interface (defaults to false) It is not possible to serve gRPC and watch NodeFeature objects at the same time. This is deliberate to avoid labeling races e.g. by nfd-worker sending gRPC requests but NodeFeature objects in the cluster "overriding" those changes (labels from the gRPC requests will get overridden when NodeFeature objects are processed). --- cmd/nfd-master/main.go | 2 + deployment/base/rbac/master-clusterrole.yaml | 1 + .../templates/clusterrole.yaml | 1 + .../templates/master.yaml | 3 + .../helm/node-feature-discovery/values.yaml | 1 + .../reference/master-commandline-reference.md | 14 ++ pkg/nfd-master/nfd-api-controller.go | 85 +++++++- pkg/nfd-master/nfd-master-internal_test.go | 24 +-- pkg/nfd-master/nfd-master.go | 203 +++++++++++++----- 9 files changed, 269 insertions(+), 65 deletions(-) diff --git a/cmd/nfd-master/main.go b/cmd/nfd-master/main.go index b18223563c..40cafd7aeb 100644 --- a/cmd/nfd-master/main.go +++ b/cmd/nfd-master/main.go @@ -94,6 +94,8 @@ func initFlags(flagset *flag.FlagSet) *master.Args { flagset.Var(&args.LabelWhiteList, "label-whitelist", "Regular expression to filter label names to publish to the Kubernetes API server. "+ "NB: the label namespace is omitted i.e. the filter is only applied to the name part after '/'.") + flagset.BoolVar(&args.EnableNodeFeatureApi, "-enable-nodefeature-api", false, + "Enable the NodeFeature CRD API for receiving node features. This will automatically disable the gRPC communication.") flagset.BoolVar(&args.NoPublish, "no-publish", false, "Do not publish feature labels") flagset.BoolVar(&args.EnableTaints, "enable-taints", false, diff --git a/deployment/base/rbac/master-clusterrole.yaml b/deployment/base/rbac/master-clusterrole.yaml index ca4b5705aa..d464e546e2 100644 --- a/deployment/base/rbac/master-clusterrole.yaml +++ b/deployment/base/rbac/master-clusterrole.yaml @@ -15,6 +15,7 @@ rules: - apiGroups: - nfd.k8s-sigs.io resources: + - nodefeatures - nodefeaturerules verbs: - get diff --git a/deployment/helm/node-feature-discovery/templates/clusterrole.yaml b/deployment/helm/node-feature-discovery/templates/clusterrole.yaml index f1c5805a13..3dd6f6f3b6 100644 --- a/deployment/helm/node-feature-discovery/templates/clusterrole.yaml +++ b/deployment/helm/node-feature-discovery/templates/clusterrole.yaml @@ -27,6 +27,7 @@ rules: - apiGroups: - nfd.k8s-sigs.io resources: + - nodefeatures - nodefeaturerules verbs: - get diff --git a/deployment/helm/node-feature-discovery/templates/master.yaml b/deployment/helm/node-feature-discovery/templates/master.yaml index 498964f8ae..03e995934b 100644 --- a/deployment/helm/node-feature-discovery/templates/master.yaml +++ b/deployment/helm/node-feature-discovery/templates/master.yaml @@ -78,6 +78,9 @@ spec: {{- if .Values.master.instance | empty | not }} - "--instance={{ .Values.master.instance }}" {{- end }} + {{- if .Values.enableNodeFeatureApi }} + - "-enable-nodefeature-api" + {{- end }} {{- if .Values.master.extraLabelNs | empty | not }} - "--extra-label-ns={{- join "," .Values.master.extraLabelNs }}" {{- end }} diff --git a/deployment/helm/node-feature-discovery/values.yaml b/deployment/helm/node-feature-discovery/values.yaml index 89df864381..85a737ce1a 100644 --- a/deployment/helm/node-feature-discovery/values.yaml +++ b/deployment/helm/node-feature-discovery/values.yaml @@ -14,6 +14,7 @@ enableNodeFeatureApi: false master: instance: + featureApi: extraLabelNs: [] resourceLabels: [] featureRulesController: null diff --git a/docs/reference/master-commandline-reference.md b/docs/reference/master-commandline-reference.md index 8542742d10..45e19dac61 100644 --- a/docs/reference/master-commandline-reference.md +++ b/docs/reference/master-commandline-reference.md @@ -149,6 +149,20 @@ nfd-master -verify-node-name -ca-file=/opt/nfd/ca.crt \ -cert-file=/opt/nfd/master.crt -key-file=/opt/nfd/master.key ``` +### -enable-nodefeature-api + +The `-enable-nodefeature-api` flag enables the NodeFeature CRD API for +receiving feature requests. This will also automatically disable the gRPC +interface. + +Default: false + +Example: + +```bash +nfd-master -enable-nodefeature-api +``` + ### -no-publish The `-no-publish` flag disables updates to the Node objects in the Kubernetes diff --git a/pkg/nfd-master/nfd-api-controller.go b/pkg/nfd-master/nfd-api-controller.go index 4a2acc46a8..84fd7d5240 100644 --- a/pkg/nfd-master/nfd-api-controller.go +++ b/pkg/nfd-master/nfd-api-controller.go @@ -32,42 +32,88 @@ import ( ) type nfdController struct { - ruleLister nfdlisters.NodeFeatureRuleLister + featureLister nfdlisters.NodeFeatureLister + ruleLister nfdlisters.NodeFeatureRuleLister stopChan chan struct{} + + updateAllNodesChan chan struct{} + updateOneNodeChan chan string } -func newNfdController(config *restclient.Config) (*nfdController, error) { +func newNfdController(config *restclient.Config, disableNodeFeature bool) (*nfdController, error) { c := &nfdController{ - stopChan: make(chan struct{}, 1), + stopChan: make(chan struct{}, 1), + updateAllNodesChan: make(chan struct{}, 1), + updateOneNodeChan: make(chan string), } nfdClient := nfdclientset.NewForConfigOrDie(config) informerFactory := nfdinformers.NewSharedInformerFactory(nfdClient, 5*time.Minute) + + // Add informer for NodeFeature objects + if !disableNodeFeature { + featureInformer := informerFactory.Nfd().V1alpha1().NodeFeatures() + if _, err := featureInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + key, _ := cache.MetaNamespaceKeyFunc(obj) + klog.V(2).Infof("NodeFeature %v added", key) + c.updateOneNode(obj) + }, + UpdateFunc: func(oldObj, newObj interface{}) { + key, _ := cache.MetaNamespaceKeyFunc(newObj) + klog.V(2).Infof("NodeFeature %v updated", key) + c.updateOneNode(newObj) + }, + DeleteFunc: func(obj interface{}) { + key, _ := cache.MetaNamespaceKeyFunc(obj) + klog.V(2).Infof("NodeFeature %v deleted", key) + c.updateOneNode(obj) + }, + }); err != nil { + return nil, err + } + c.featureLister = featureInformer.Lister() + } + + // Add informer for NodeFeatureRule objects ruleInformer := informerFactory.Nfd().V1alpha1().NodeFeatureRules() if _, err := ruleInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(object interface{}) { key, _ := cache.MetaNamespaceKeyFunc(object) klog.V(2).Infof("NodeFeatureRule %v added", key) + if !disableNodeFeature { + c.updateAllNodes() + } + // else: rules will be processed only when gRPC requests are received }, UpdateFunc: func(oldObject, newObject interface{}) { key, _ := cache.MetaNamespaceKeyFunc(newObject) klog.V(2).Infof("NodeFeatureRule %v updated", key) + if !disableNodeFeature { + c.updateAllNodes() + } + // else: rules will be processed only when gRPC requests are received }, DeleteFunc: func(object interface{}) { key, _ := cache.MetaNamespaceKeyFunc(object) klog.V(2).Infof("NodeFeatureRule %v deleted", key) + if !disableNodeFeature { + c.updateAllNodes() + } + // else: rules will be processed only when gRPC requests are received }, }); err != nil { return nil, err } + c.ruleLister = ruleInformer.Lister() + + // Start informers informerFactory.Start(c.stopChan) utilruntime.Must(nfdv1alpha1.AddToScheme(nfdscheme.Scheme)) - c.ruleLister = ruleInformer.Lister() - return c, nil } @@ -77,3 +123,32 @@ func (c *nfdController) stop() { default: } } + +func (c *nfdController) updateOneNode(obj interface{}) { + o, ok := obj.(*nfdv1alpha1.NodeFeature) + if !ok { + klog.Errorf("not a NodeFeature object (but of type %T): %v", obj, obj) + return + } + + nodeName, ok := o.Labels[nfdv1alpha1.NodeFeatureObjNodeNameLabel] + if !ok { + klog.Errorf("no node name for NodeFeature object %s/%s: %q label is missing", + o.Namespace, o.Name, nfdv1alpha1.NodeFeatureObjNodeNameLabel) + return + } + if nodeName == "" { + klog.Errorf("no node name for NodeFeature object %s/%s: %q label is empty", + o.Namespace, o.Name, nfdv1alpha1.NodeFeatureObjNodeNameLabel) + return + } + + c.updateOneNodeChan <- nodeName +} + +func (c *nfdController) updateAllNodes() { + select { + case c.updateAllNodesChan <- struct{}{}: + default: + } +} diff --git a/pkg/nfd-master/nfd-master-internal_test.go b/pkg/nfd-master/nfd-master-internal_test.go index fc522daece..cf50e49064 100644 --- a/pkg/nfd-master/nfd-master-internal_test.go +++ b/pkg/nfd-master/nfd-master-internal_test.go @@ -61,7 +61,7 @@ func newMockMaster(apihelper apihelper.APIHelpers) *nfdMaster { } } -func TestUpdateNodeFeatures(t *testing.T) { +func TestUpdateNodeObject(t *testing.T) { Convey("When I update the node using fake client", t, func() { fakeFeatureLabels := map[string]string{ nfdv1alpha1.FeatureLabelNs + "/source-feature.1": "1", @@ -112,10 +112,10 @@ func TestUpdateNodeFeatures(t *testing.T) { } mockAPIHelper.On("GetClient").Return(mockClient, nil) - mockAPIHelper.On("GetNode", mockClient, mockNodeName).Return(mockNode, nil).Once() + mockAPIHelper.On("GetNode", mockClient, mockNodeName).Return(mockNode, nil).Twice() mockAPIHelper.On("PatchNode", mockClient, mockNodeName, mock.MatchedBy(jsonPatchMatcher(metadataPatches))).Return(nil) mockAPIHelper.On("PatchNodeStatus", mockClient, mockNodeName, mock.MatchedBy(jsonPatchMatcher(statusPatches))).Return(nil) - err := mockMaster.updateNodeFeatures(mockClient, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources) + err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources, nil) Convey("Error is nil", func() { So(err, ShouldBeNil) @@ -125,7 +125,7 @@ func TestUpdateNodeFeatures(t *testing.T) { Convey("When I fail to update the node with feature labels", func() { expectedError := fmt.Errorf("no client is passed, client: ") mockAPIHelper.On("GetClient").Return(nil, expectedError) - err := mockMaster.updateNodeFeatures(nil, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources) + err := mockMaster.updateNodeObject(nil, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources, nil) Convey("Error is produced", func() { So(err, ShouldResemble, expectedError) @@ -135,7 +135,7 @@ func TestUpdateNodeFeatures(t *testing.T) { Convey("When I fail to get a mock client while updating feature labels", func() { expectedError := fmt.Errorf("no client is passed, client: ") mockAPIHelper.On("GetClient").Return(nil, expectedError) - err := mockMaster.updateNodeFeatures(nil, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources) + err := mockMaster.updateNodeObject(nil, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources, nil) Convey("Error is produced", func() { So(err, ShouldResemble, expectedError) @@ -145,8 +145,8 @@ func TestUpdateNodeFeatures(t *testing.T) { Convey("When I fail to get a mock node while updating feature labels", func() { expectedError := errors.New("fake error") mockAPIHelper.On("GetClient").Return(mockClient, nil) - mockAPIHelper.On("GetNode", mockClient, mockNodeName).Return(nil, expectedError).Once() - err := mockMaster.updateNodeFeatures(mockClient, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources) + mockAPIHelper.On("GetNode", mockClient, mockNodeName).Return(nil, expectedError).Twice() + err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources, nil) Convey("Error is produced", func() { So(err, ShouldEqual, expectedError) @@ -156,9 +156,9 @@ func TestUpdateNodeFeatures(t *testing.T) { Convey("When I fail to update a mock node while updating feature labels", func() { expectedError := errors.New("fake error") mockAPIHelper.On("GetClient").Return(mockClient, nil) - mockAPIHelper.On("GetNode", mockClient, mockNodeName).Return(mockNode, nil).Once() - mockAPIHelper.On("PatchNode", mockClient, mockNodeName, mock.Anything).Return(expectedError).Once() - err := mockMaster.updateNodeFeatures(mockClient, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources) + mockAPIHelper.On("GetNode", mockClient, mockNodeName).Return(mockNode, nil).Twice() + mockAPIHelper.On("PatchNode", mockClient, mockNodeName, mock.Anything).Return(expectedError).Twice() + err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources, nil) Convey("Error is produced", func() { So(err.Error(), ShouldEndWith, expectedError.Error()) @@ -294,7 +294,7 @@ func TestRemovingExtResources(t *testing.T) { func TestSetLabels(t *testing.T) { Convey("When servicing SetLabels request", t, func() { - const workerName = "mock-worker" + const workerName = mockNodeName const workerVer = "0.1-test" mockHelper := &apihelper.MockAPIHelpers{} mockMaster := newMockMaster(mockHelper) @@ -324,7 +324,7 @@ func TestSetLabels(t *testing.T) { } mockHelper.On("GetClient").Return(mockClient, nil) - mockHelper.On("GetNode", mockClient, workerName).Return(mockNode, nil) + mockHelper.On("GetNode", mockClient, workerName).Return(mockNode, nil).Twice() mockHelper.On("PatchNode", mockClient, mockNodeName, mock.MatchedBy(jsonPatchMatcher(expectedPatches))).Return(nil) mockHelper.On("PatchNodeStatus", mockClient, mockNodeName, mock.MatchedBy(jsonPatchMatcher(expectedStatusPatches))).Return(nil) _, err := mockMaster.SetLabels(mockCtx, mockReq) diff --git a/pkg/nfd-master/nfd-master.go b/pkg/nfd-master/nfd-master.go index db4f9f38e2..afc9719e39 100644 --- a/pkg/nfd-master/nfd-master.go +++ b/pkg/nfd-master/nfd-master.go @@ -36,6 +36,7 @@ import ( "google.golang.org/grpc/health/grpc_health_v1" "google.golang.org/grpc/peer" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" label "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" @@ -69,6 +70,7 @@ type Args struct { Kubeconfig string LabelWhiteList utils.RegexpVal FeatureRulesController bool + EnableNodeFeatureApi bool NoPublish bool EnableTaints bool Port int @@ -87,6 +89,7 @@ type nfdMaster struct { *nfdController args Args + namespace string nodeName string server *grpc.Server stop chan struct{} @@ -98,9 +101,10 @@ type nfdMaster struct { // NewNfdMaster creates a new NfdMaster server instance. func NewNfdMaster(args *Args) (NfdMaster, error) { nfd := &nfdMaster{args: *args, - nodeName: os.Getenv("NODE_NAME"), - ready: make(chan bool, 1), - stop: make(chan struct{}, 1), + nodeName: os.Getenv("NODE_NAME"), + namespace: utils.GetKubernetesNamespace(), + ready: make(chan bool, 1), + stop: make(chan struct{}, 1), } if args.Instance != "" { @@ -144,6 +148,7 @@ func (m *nfdMaster) Run() error { klog.Infof("Master instance: %q", m.args.Instance) } klog.Infof("NodeName: %q", m.nodeName) + klog.Infof("Kubernetes namespace: %q", m.namespace) if m.args.Prune { return m.prune() @@ -155,7 +160,7 @@ func (m *nfdMaster) Run() error { return err } klog.Info("starting nfd api controller") - m.nfdController, err = newNfdController(kubeconfig) + m.nfdController, err = newNfdController(kubeconfig, !m.args.EnableNodeFeatureApi) if err != nil { return fmt.Errorf("failed to initialize CRD controller: %w", err) } @@ -170,7 +175,14 @@ func (m *nfdMaster) Run() error { // Run gRPC server grpcErr := make(chan error, 1) - go m.runGrpcServer(grpcErr) + if !m.args.EnableNodeFeatureApi { + go m.runGrpcServer(grpcErr) + } + + // Run updater that handles events from the nfd CRD API. + if m.nfdController != nil { + go m.nfdAPIUpdateHandler() + } // Notify that we're ready to accept connections m.ready <- true @@ -245,6 +257,22 @@ func (m *nfdMaster) runGrpcServer(errChan chan<- error) { } } +// nfdAPIUpdateHandler handles events from the nfd API controller. +func (m *nfdMaster) nfdAPIUpdateHandler() { + for { + select { + case <-m.nfdController.updateAllNodesChan: + if err := m.nfdAPIUpdateAllNodes(); err != nil { + klog.Error(err) + } + case nodeName := <-m.nfdController.updateOneNodeChan: + if err := m.nfdAPIUpdateOneNode(nodeName); err != nil { + klog.Error(err) + } + } + } +} + // Stop NfdMaster func (m *nfdMaster) Stop() { m.server.GracefulStop() @@ -290,16 +318,9 @@ func (m *nfdMaster) prune() error { klog.Infof("pruning node %q...", node.Name) // Prune labels and extended resources - err := m.updateNodeFeatures(cli, node.Name, Labels{}, Annotations{}, ExtendedResources{}) + err := m.updateNodeObject(cli, node.Name, Labels{}, Annotations{}, ExtendedResources{}, []corev1.Taint{}) if err != nil { - return fmt.Errorf("failed to prune labels from node %q: %v", node.Name, err) - } - - // Prune taints - err = m.setTaints(cli, []corev1.Taint{}, node.Name) - - if err != nil { - return fmt.Errorf("failed to prune taints from node %q: %v", node.Name, err) + return fmt.Errorf("failed to prune node %q: %v", node.Name, err) } // Prune annotations @@ -421,20 +442,6 @@ func (m *nfdMaster) SetLabels(c context.Context, r *pb.SetLabelsRequest) (*pb.Se klog.Infof("received labeling request for node %q", r.NodeName) } - // Mix in CR-originated labels - rawLabels := make(map[string]string) - if r.Labels != nil { - // NOTE: we effectively mangle the request struct by not creating a deep copy of the map - rawLabels = r.Labels - } - crLabels, crTaints := m.processNodeFeatureRule(r) - - for k, v := range crLabels { - rawLabels[k] = v - } - - labels, extendedResources := filterFeatureLabels(rawLabels, m.args.ExtraLabelNs, m.args.LabelWhiteList.Regexp, m.args.ResourceLabels) - if !m.args.NoPublish { cli, err := m.apihelper.GetClient() if err != nil { @@ -444,27 +451,124 @@ func (m *nfdMaster) SetLabels(c context.Context, r *pb.SetLabelsRequest) (*pb.Se // Advertise NFD worker version as an annotation annotations := Annotations{m.instanceAnnotation(nfdv1alpha1.WorkerVersionAnnotation): r.NfdVersion} - err = m.updateNodeFeatures(cli, r.NodeName, labels, annotations, extendedResources) - if err != nil { - klog.Errorf("failed to advertise labels: %v", err) + // Create labels et al + if err := m.refreshNodeFeatures(cli, r.NodeName, annotations, r.GetLabels(), r.GetFeatures()); err != nil { return &pb.SetLabelsReply{}, err } + } + return &pb.SetLabelsReply{}, nil +} + +func (m *nfdMaster) nfdAPIUpdateAllNodes() error { + klog.Info("will process all nodes in the cluster") + + cli, err := m.apihelper.GetClient() + if err != nil { + return err + } - // set taints - var taints []corev1.Taint - if m.args.EnableTaints { - taints = crTaints + nodes, err := m.apihelper.GetNodes(cli) + if err != nil { + return err + } + + for _, node := range nodes.Items { + if err := m.nfdAPIUpdateOneNode(node.Name); err != nil { + return err } + } - // Call setTaints even though the feature flag is disabled. This - // ensures that we delete NFD owned stale taints when flag got - // turned off. - err = m.setTaints(cli, taints, r.NodeName) - if err != nil { - return &pb.SetLabelsReply{}, err + return nil +} + +func (m *nfdMaster) nfdAPIUpdateOneNode(nodeName string) error { + sel := labels.SelectorFromSet(labels.Set{nfdv1alpha1.NodeFeatureObjNodeNameLabel: nodeName}) + objs, err := m.nfdController.featureLister.List(sel) + if len(objs) == 0 { + klog.Infof("no NodeFeature object exists for node %q, skipping...", nodeName) + return nil + } else if err != nil { + return fmt.Errorf("failed to get NodeFeature resources for node %q: %w", nodeName, err) + } + + // Sort our objects + sort.Slice(objs, func(i, j int) bool { + // Objects in our nfd namespace gets into the beginning of the list + if objs[i].Namespace == m.namespace && objs[j].Namespace != m.namespace { + return true } + if objs[i].Namespace != m.namespace && objs[j].Namespace == m.namespace { + return false + } + // After the nfd namespace, sort objects by their name + if objs[i].Name != objs[j].Name { + return objs[i].Name < objs[j].Name + } + // Objects with the same name are sorted by their namespace + return objs[i].Namespace < objs[j].Namespace + }) + + if m.args.NoPublish { + return nil } - return &pb.SetLabelsReply{}, nil + + klog.V(1).Infof("processing node %q, initiated by NodeFeature API", nodeName) + + // Merge in features + // + // TODO: support multiple NodeFeature objects. There are two obvious options to implement this: + // 1. Merge features of all objects into one joint object + // 2. Change the rule api to support handle multiple objects + // Of these #2 would probably perform better with lot less data to copy. We + // could probably even get rid of the DeepCopy in this scenario. + features := objs[0].DeepCopy() + + annotations := Annotations{} + if objs[0].Namespace == m.namespace && objs[0].Name == nodeName { + // This is the one created by nfd-worker + if v := objs[0].Annotations[nfdv1alpha1.WorkerVersionAnnotation]; v != "" { + annotations[nfdv1alpha1.WorkerVersionAnnotation] = v + } + } + + // Create labels et al + cli, err := m.apihelper.GetClient() + if err != nil { + return err + } + if err := m.refreshNodeFeatures(cli, nodeName, annotations, features.Spec.Labels, &features.Spec.Features); err != nil { + return err + } + + return nil +} + +func (m *nfdMaster) refreshNodeFeatures(cli *kubernetes.Clientset, nodeName string, annotations, labels map[string]string, features *nfdv1alpha1.Features) error { + if labels == nil { + labels = make(map[string]string) + } + + crLabels, crTaints := m.processNodeFeatureRule(features) + + // Mix in CR-originated labels + for k, v := range crLabels { + labels[k] = v + } + + labels, extendedResources := filterFeatureLabels(labels, m.args.ExtraLabelNs, m.args.LabelWhiteList.Regexp, m.args.ResourceLabels) + + var taints []corev1.Taint + if m.args.EnableTaints { + taints = crTaints + } + + err := m.updateNodeObject(cli, nodeName, labels, annotations, extendedResources, taints) + if err != nil { + klog.Errorf("failed to update node %q: %v", nodeName, err) + return err + } + + return nil } // setTaints sets node taints and annotations based on the taints passed via @@ -572,7 +676,7 @@ func authorizeClient(c context.Context, checkNodeName bool, nodeName string) err return nil } -func (m *nfdMaster) processNodeFeatureRule(r *pb.SetLabelsRequest) (map[string]string, []corev1.Taint) { +func (m *nfdMaster) processNodeFeatureRule(features *nfdv1alpha1.Features) (map[string]string, []corev1.Taint) { if m.nfdController == nil { return nil, nil } @@ -589,9 +693,6 @@ func (m *nfdMaster) processNodeFeatureRule(r *pb.SetLabelsRequest) (map[string]s return nil, nil } - // Helper struct for rule processing - features := r.GetFeatures() - // Process all rule CRs for _, spec := range ruleSpecs { switch { @@ -621,10 +722,10 @@ func (m *nfdMaster) processNodeFeatureRule(r *pb.SetLabelsRequest) (map[string]s return labels, taints } -// updateNodeFeatures ensures the Kubernetes node object is up to date, +// updateNodeObject ensures the Kubernetes node object is up to date, // creating new labels and extended resources where necessary and removing // outdated ones. Also updates the corresponding annotations. -func (m *nfdMaster) updateNodeFeatures(cli *kubernetes.Clientset, nodeName string, labels Labels, annotations Annotations, extendedResources ExtendedResources) error { +func (m *nfdMaster) updateNodeObject(cli *kubernetes.Clientset, nodeName string, labels Labels, annotations Annotations, extendedResources ExtendedResources, taints []corev1.Taint) error { if cli == nil { return fmt.Errorf("no client is passed, client: %v", cli) } @@ -677,6 +778,12 @@ func (m *nfdMaster) updateNodeFeatures(cli *kubernetes.Clientset, nodeName strin klog.V(1).Infof("no updates to node %q", nodeName) } + // Set taints + err = m.setTaints(cli, taints, node.Name) + if err != nil { + return err + } + return err } From 5a717c418b50bfaabe8776895a8cb8cd8693725c Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Tue, 13 Dec 2022 17:14:30 +0200 Subject: [PATCH 46/87] docs: small reordering of master cmdline reference Move documentation of -enable-taints near '-enable-nodefeature-api' and '-no-publish' as they are related in that they control the enablement of APIs. --- .../reference/master-commandline-reference.md | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/reference/master-commandline-reference.md b/docs/reference/master-commandline-reference.md index 45e19dac61..c7dae1d60d 100644 --- a/docs/reference/master-commandline-reference.md +++ b/docs/reference/master-commandline-reference.md @@ -99,18 +99,6 @@ Example: nfd-master -cert-file=/opt/nfd/master.crt -key-file=/opt/nfd/master.key -ca-file=/opt/nfd/ca.crt ``` -### -enable-taints - -The `-enable-taints` flag enables/disables node tainting feature of NFD. - -Default: *false* - -Example: - -```bash -nfd-master -enable-taints=true -``` - ### -key-file The `-key-file` is one of the three flags (together with `-ca-file` and @@ -163,6 +151,18 @@ Example: nfd-master -enable-nodefeature-api ``` +### -enable-taints + +The `-enable-taints` flag enables/disables node tainting feature of NFD. + +Default: *false* + +Example: + +```bash +nfd-master -enable-taints=true +``` + ### -no-publish The `-no-publish` flag disables updates to the Node objects in the Kubernetes From 9f0806593d39b0793beec41b23534c0a023939eb Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Tue, 22 Nov 2022 11:51:20 +0200 Subject: [PATCH 47/87] nfd-master: rename -featurerules-controller flag to -crd-controller Deprecate the '-featurerules-controller' command line flag as the name does not describe the functionality anymore: in practice it controls the CRD controller handling both NodeFeature and NodeFeatureRule objects. The patch introduces a duplicate, more generally named, flag '-crd-controller'. A warning is printed in the log if '-featurerules-controller' flag is encountered. --- cmd/nfd-master/main.go | 14 ++++++-- .../templates/master.yaml | 9 ++++-- .../helm/node-feature-discovery/values.yaml | 1 + docs/deployment/helm.md | 3 +- .../reference/master-commandline-reference.md | 14 +++++--- pkg/nfd-master/nfd-master.go | 32 +++++++++---------- 6 files changed, 46 insertions(+), 27 deletions(-) diff --git a/cmd/nfd-master/main.go b/cmd/nfd-master/main.go index 40cafd7aeb..c929ec551e 100644 --- a/cmd/nfd-master/main.go +++ b/cmd/nfd-master/main.go @@ -50,6 +50,14 @@ func main() { os.Exit(2) } + // Check deprecated flags + flags.Visit(func(f *flag.Flag) { + switch f.Name { + case "featurerules-controller": + klog.Warningf("-featurerules-controller is deprecated, use '-crd-controller' flag instead") + } + }) + if *printVersion { fmt.Println(ProgramName, version.Get()) os.Exit(0) @@ -100,8 +108,10 @@ func initFlags(flagset *flag.FlagSet) *master.Args { "Do not publish feature labels") flagset.BoolVar(&args.EnableTaints, "enable-taints", false, "Enable node tainting feature") - flagset.BoolVar(&args.FeatureRulesController, "featurerules-controller", true, - "Enable controller for NodeFeatureRule objects. Generates node labels based on the rules in these CRs.") + flagset.BoolVar(&args.CrdController, "featurerules-controller", true, + "Enable NFD CRD API controller. DEPRECATED: use -crd-controller instead") + flagset.BoolVar(&args.CrdController, "crd-controller", true, + "Enable NFD CRD API controller for processing NodeFeature and NodeFeatureRule objects.") flagset.IntVar(&args.Port, "port", 8080, "Port on which to listen for connections.") flagset.BoolVar(&args.Prune, "prune", false, diff --git a/deployment/helm/node-feature-discovery/templates/master.yaml b/deployment/helm/node-feature-discovery/templates/master.yaml index 03e995934b..26a326cb80 100644 --- a/deployment/helm/node-feature-discovery/templates/master.yaml +++ b/deployment/helm/node-feature-discovery/templates/master.yaml @@ -87,12 +87,15 @@ spec: {{- if .Values.master.resourceLabels | empty | not }} - "--resource-labels={{- join "," .Values.master.resourceLabels }}" {{- end }} - {{- if .Values.master.featureRulesController | kindIs "invalid" | not }} - - "-featurerules-controller={{ .Values.master.featureRulesController }}" + {{- if .Values.master.crdController | kindIs "invalid" | not }} + - "-crd-controller={{ .Values.master.crdController }}" {{- else }} - ## By default, disable NodeFeatureRules controller for other than the default instances + ## By default, disable crd controller for other than the default instances - "-featurerules-controller={{ .Values.master.instance | empty }}" {{- end }} + {{- if .Values.master.featureRulesController | kindIs "invalid" | not }} + - "-featurerules-controller={{ .Values.master.featureRulesController }}" + {{- end }} {{- if .Values.tls.enable }} - "--ca-file=/etc/kubernetes/node-feature-discovery/certs/ca.crt" - "--key-file=/etc/kubernetes/node-feature-discovery/certs/tls.key" diff --git a/deployment/helm/node-feature-discovery/values.yaml b/deployment/helm/node-feature-discovery/values.yaml index 85a737ce1a..8531e963af 100644 --- a/deployment/helm/node-feature-discovery/values.yaml +++ b/deployment/helm/node-feature-discovery/values.yaml @@ -17,6 +17,7 @@ master: featureApi: extraLabelNs: [] resourceLabels: [] + crdController: null featureRulesController: null deploymentAnnotations: {} replicaCount: 1 diff --git a/docs/deployment/helm.md b/docs/deployment/helm.md index 6e032026e1..03115988e9 100644 --- a/docs/deployment/helm.md +++ b/docs/deployment/helm.md @@ -107,7 +107,8 @@ We have introduced the following Chart parameters. | `master.instance` | string | | Instance name. Used to separate annotation namespaces for multiple parallel deployments | | `master.extraLabelNs` | array | [] | List of allowed extra label namespaces | | `master.resourceLabels` | array | [] | List of labels to be registered as extended resources | -| `master.featureRulesController` | bool | null | Specifies whether the controller for processing of NodeFeatureRule objects is enabled. If not set, controller will be enabled if `master.instance` is empty. | +| `master.crdController` | bool | null | Specifies whether the NFD CRD API controller is enabled. If not set, controller will be enabled if `master.instance` is empty. | +| `master.featureRulesController` | bool | null | DEPRECATED: use `master.crdController` instead | | `master.replicaCount` | integer | 1 | Number of desired pods. This is a pointer to distinguish between explicit zero and not specified | | `master.podSecurityContext` | dict | {} | [PodSecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) holds pod-level security attributes and common container settings | | `master.securityContext` | dict | {} | Container [security settings](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container)| diff --git a/docs/reference/master-commandline-reference.md b/docs/reference/master-commandline-reference.md index c7dae1d60d..6bda05d505 100644 --- a/docs/reference/master-commandline-reference.md +++ b/docs/reference/master-commandline-reference.md @@ -177,20 +177,24 @@ Example: nfd-master -no-publish ``` -### -featurerules-controller +### -crd-controller -The `-featurerules-controller` flag controlers the processing of -NodeFeatureRule objects, effectively enabling/disabling labels from these -custom labeling rules. +The `-crd-controller` flag specifies whether the NFD CRD API controller is +enabled or not. The controller is responsible for processing NodeFeature and +NodeFeatureRule objects. Default: *true* Example: ```bash -nfd-master -featurerules-controller=false +nfd-master -crd-controller=false ``` +### -featurerules-controller + +**DEPRECATED**: use [`-crd-controller`](#-crd-controller) instead. + ### -label-whitelist The `-label-whitelist` specifies a regular expression for filtering feature diff --git a/pkg/nfd-master/nfd-master.go b/pkg/nfd-master/nfd-master.go index afc9719e39..7d1bdac3e2 100644 --- a/pkg/nfd-master/nfd-master.go +++ b/pkg/nfd-master/nfd-master.go @@ -62,21 +62,21 @@ type Annotations map[string]string // Args holds command line arguments type Args struct { - CaFile string - CertFile string - ExtraLabelNs utils.StringSetVal - Instance string - KeyFile string - Kubeconfig string - LabelWhiteList utils.RegexpVal - FeatureRulesController bool - EnableNodeFeatureApi bool - NoPublish bool - EnableTaints bool - Port int - Prune bool - VerifyNodeName bool - ResourceLabels utils.StringSetVal + CaFile string + CertFile string + ExtraLabelNs utils.StringSetVal + Instance string + KeyFile string + Kubeconfig string + LabelWhiteList utils.RegexpVal + CrdController bool + EnableNodeFeatureApi bool + NoPublish bool + EnableTaints bool + Port int + Prune bool + VerifyNodeName bool + ResourceLabels utils.StringSetVal } type NfdMaster interface { @@ -154,7 +154,7 @@ func (m *nfdMaster) Run() error { return m.prune() } - if m.args.FeatureRulesController { + if m.args.CrdController { kubeconfig, err := m.getKubeconfig() if err != nil { return err From 81b0945ced2b1eaf402f24833c0fecf38dcf4da0 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Wed, 14 Dec 2022 12:51:14 +0200 Subject: [PATCH 48/87] nfd-master: fix creation of the -enable-nodefeature-api flag Extra dash caused a panic when trying to run the binary. --- cmd/nfd-master/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/nfd-master/main.go b/cmd/nfd-master/main.go index 40cafd7aeb..958d310c1e 100644 --- a/cmd/nfd-master/main.go +++ b/cmd/nfd-master/main.go @@ -94,7 +94,7 @@ func initFlags(flagset *flag.FlagSet) *master.Args { flagset.Var(&args.LabelWhiteList, "label-whitelist", "Regular expression to filter label names to publish to the Kubernetes API server. "+ "NB: the label namespace is omitted i.e. the filter is only applied to the name part after '/'.") - flagset.BoolVar(&args.EnableNodeFeatureApi, "-enable-nodefeature-api", false, + flagset.BoolVar(&args.EnableNodeFeatureApi, "enable-nodefeature-api", false, "Enable the NodeFeature CRD API for receiving node features. This will automatically disable the gRPC communication.") flagset.BoolVar(&args.NoPublish, "no-publish", false, "Do not publish feature labels") From 13f3959ebef66bfc1d14a769103c965a89220c0d Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Wed, 14 Dec 2022 14:14:49 +0200 Subject: [PATCH 49/87] test/e2e: fix creation of NFD CRDs After introducing NodeFeatureRule we packed two CRD definitions in one yaml file. Our e2e-tests were not prepared to that and the file itself was also renamed so it couldn't even be read by the test suite. With this change the e2e-tests start to create NodeFeatre CRD in the test cluster, preparing for the addition of e2e-tests for NodeFeature API. --- test/e2e/node_feature_discovery.go | 16 ++--- test/e2e/utils/crd.go | 94 +++++++++++++++++++++--------- 2 files changed, 74 insertions(+), 36 deletions(-) diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index 39c50ccad3..e19144916a 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -390,7 +390,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { Context("and nfd-worker and NodeFeatureRules objects deployed", func() { var extClient *extclient.Clientset var nfdClient *nfdclient.Clientset - var crd *apiextensionsv1.CustomResourceDefinition + var crds []*apiextensionsv1.CustomResourceDefinition BeforeEach(func() { // Create clients for apiextensions and our CRD api @@ -398,15 +398,17 @@ var _ = SIGDescribe("Node Feature Discovery", func() { nfdClient = nfdclient.NewForConfigOrDie(f.ClientConfig()) // Create CRDs - By("Creating NodeFeatureRule CRD") + By("Creating NFD CRDs") var err error - crd, err = testutils.CreateNodeFeatureRulesCRD(extClient) + crds, err = testutils.CreateNfdCRDs(extClient) Expect(err).NotTo(HaveOccurred()) }) AfterEach(func() { - err := extClient.ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), crd.Name, metav1.DeleteOptions{}) - Expect(err).NotTo(HaveOccurred()) + for _, crd := range crds { + err := extClient.ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), crd.Name, metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + } }) It("custom labels from the NodeFeatureRule rules should be created", func() { @@ -438,13 +440,13 @@ core: "feature.node.kubernetes.io/e2e-instance-test-1": "true"} By("Creating NodeFeatureRules #1") - Expect(testutils.CreateNodeFeatureRuleFromFile(nfdClient, "nodefeaturerule-1.yaml")).NotTo(HaveOccurred()) + Expect(testutils.CreateNodeFeatureRulesFromFile(nfdClient, "nodefeaturerule-1.yaml")).NotTo(HaveOccurred()) By("Verifying node labels from NodeFeatureRules #1") Expect(waitForNfdNodeLabels(f.ClientSet, expected)).NotTo(HaveOccurred()) By("Creating NodeFeatureRules #2") - Expect(testutils.CreateNodeFeatureRuleFromFile(nfdClient, "nodefeaturerule-2.yaml")).NotTo(HaveOccurred()) + Expect(testutils.CreateNodeFeatureRulesFromFile(nfdClient, "nodefeaturerule-2.yaml")).NotTo(HaveOccurred()) // Add features from NodeFeatureRule #2 expected["feature.node.kubernetes.io/e2e-matchany-test-1"] = "true" diff --git a/test/e2e/utils/crd.go b/test/e2e/utils/crd.go index 5a45e642db..76ea53c00d 100644 --- a/test/e2e/utils/crd.go +++ b/test/e2e/utils/crd.go @@ -17,6 +17,7 @@ limitations under the License. package utils import ( + "bytes" "context" "fmt" "os" @@ -36,69 +37,104 @@ import ( var packagePath string -// CreateNodeFeatureRulesCRD creates the NodeFeatureRule CRD in the API server. -func CreateNodeFeatureRulesCRD(cli extclient.Interface) (*apiextensionsv1.CustomResourceDefinition, error) { - crd, err := crdFromFile(filepath.Join(packagePath, "..", "..", "..", "deployment", "base", "nfd-crds", "nodefeaturerule-crd.yaml")) +// CreateNfdCRDs creates the NodeFeatureRule CRD in the API server. +func CreateNfdCRDs(cli extclient.Interface) ([]*apiextensionsv1.CustomResourceDefinition, error) { + crds, err := crdsFromFile(filepath.Join(packagePath, "..", "..", "..", "deployment", "base", "nfd-crds", "nfd-api-crds.yaml")) if err != nil { return nil, err } - // Delete existing CRD (if any) with this we also get rid of stale objects - err = cli.ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), crd.Name, metav1.DeleteOptions{}) - if err != nil && !errors.IsNotFound(err) { - return nil, fmt.Errorf("failed to delete NodeFeatureRule CRD: %w", err) + newCRDs := make([]*apiextensionsv1.CustomResourceDefinition, len(crds)) + for i, crd := range crds { + // Delete existing CRD (if any) with this we also get rid of stale objects + err = cli.ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), crd.Name, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + return nil, fmt.Errorf("failed to delete NodeFeatureRule CRD: %w", err) + } + newCRDs[i], err = cli.ApiextensionsV1().CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{}) + if err != nil { + return nil, err + } } - - return cli.ApiextensionsV1().CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{}) + return newCRDs, nil } // CreateNodeFeatureRuleFromFile creates a NodeFeatureRule object from a given file located under test data directory. -func CreateNodeFeatureRuleFromFile(cli nfdclientset.Interface, filename string) error { - obj, err := nodeFeatureRuleFromFile(filepath.Join(packagePath, "..", "data", filename)) +func CreateNodeFeatureRulesFromFile(cli nfdclientset.Interface, filename string) error { + objs, err := nodeFeatureRulesFromFile(filepath.Join(packagePath, "..", "data", filename)) if err != nil { return err } - _, err = cli.NfdV1alpha1().NodeFeatureRules().Create(context.TODO(), obj, metav1.CreateOptions{}) - return err + + for _, obj := range objs { + if _, err = cli.NfdV1alpha1().NodeFeatureRules().Create(context.TODO(), obj, metav1.CreateOptions{}); err != nil { + return err + } + } + return nil } -func apiObjFromFile(path string, decoder apiruntime.Decoder) (apiruntime.Object, error) { +func apiObjsFromFile(path string, decoder apiruntime.Decoder) ([]apiruntime.Object, error) { data, err := os.ReadFile(path) if err != nil { return nil, err } - obj, _, err := decoder.Decode(data, nil, nil) - return obj, err + // TODO: find out a nicer way to decode multiple api objects from a single + // file (K8s must have that somewhere) + split := bytes.Split(data, []byte("---")) + objs := []apiruntime.Object{} + + for _, slice := range split { + if len(slice) == 0 { + continue + } + obj, _, err := decoder.Decode(slice, nil, nil) + if err != nil { + return nil, err + } + objs = append(objs, obj) + } + return objs, err } -// crdFromFile creates a CustomResourceDefinition API object from a file. -func crdFromFile(path string) (*apiextensionsv1.CustomResourceDefinition, error) { - obj, err := apiObjFromFile(path, scheme.Codecs.UniversalDeserializer()) +// crdsFromFile creates a CustomResourceDefinition API object from a file. +func crdsFromFile(path string) ([]*apiextensionsv1.CustomResourceDefinition, error) { + objs, err := apiObjsFromFile(path, scheme.Codecs.UniversalDeserializer()) if err != nil { return nil, err } - crd, ok := obj.(*apiextensionsv1.CustomResourceDefinition) - if !ok { - return nil, fmt.Errorf("unexpected type %t when reading %q", obj, path) + crds := make([]*apiextensionsv1.CustomResourceDefinition, len(objs)) + + for i, obj := range objs { + var ok bool + crds[i], ok = obj.(*apiextensionsv1.CustomResourceDefinition) + if !ok { + return nil, fmt.Errorf("unexpected type %T when reading %q", obj, path) + } } - return crd, nil + return crds, nil } -func nodeFeatureRuleFromFile(path string) (*nfdv1alpha1.NodeFeatureRule, error) { - obj, err := apiObjFromFile(path, nfdscheme.Codecs.UniversalDeserializer()) +func nodeFeatureRulesFromFile(path string) ([]*nfdv1alpha1.NodeFeatureRule, error) { + objs, err := apiObjsFromFile(path, nfdscheme.Codecs.UniversalDeserializer()) if err != nil { return nil, err } - crd, ok := obj.(*nfdv1alpha1.NodeFeatureRule) - if !ok { - return nil, fmt.Errorf("unexpected type %t when reading %q", obj, path) + crs := make([]*nfdv1alpha1.NodeFeatureRule, len(objs)) + + for i, obj := range objs { + var ok bool + crs[i], ok = obj.(*nfdv1alpha1.NodeFeatureRule) + if !ok { + return nil, fmt.Errorf("unexpected type %t when reading %q", obj, path) + } } - return crd, nil + return crs, nil } func init() { From 79ed747be8aef175efdd7b6416ca26c689d68de5 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Wed, 21 Sep 2022 18:57:53 +0300 Subject: [PATCH 50/87] nfd-master: handle multiple NodeFeature objects Implement handling of multiple NodeFeature objects by merging all objects (targeting a certain node) into one before processing the data. This patch implements MergeInto() methods for all required data types. With support for multiple NodeFeature objects per node, The "nfd api workflow" can be easily demonstrated and tested from the command line. Creating the folloiwing object (assuming node-n exists in the cluster): apiVersion: nfd.k8s-sigs.io/v1alpha1 kind: NodeFeature metadata: labels: nfd.node.kubernetes.io/node-name: node-n name: my-features-for-node-n spec: # Features for NodeFeatureRule matching features: flags: vendor.domain-a: elements: feature-x: {} attributes: vendor.domain-b: elements: feature-y: "foo" feature-z: "123" instances: vendor.domain-c: elements: - attributes: name: "elem-1" vendor: "acme" - attributes: name: "elem-2" vendor: "acme" # Labels to be created labels: vendor-feature.enabled: "true" vendor-setting.value: "100" will create two feature labes: feature.node.kubernetes.io/vendor-feature.enabled: "true" feature.node.kubernetes.io/vendor-setting.value: "100" In addition it will advertise hidden/raw features that can be used for custom rules in NodeFeatureRule objects. Now, creating a NodeFeatureRule object: apiVersion: nfd.k8s-sigs.io/v1alpha1 kind: NodeFeatureRule metadata: name: my-rule spec: rules: - name: "my feature rule" labels: "my-feature": "true" matchFeatures: - feature: vendor.domain-a matchExpressions: feature-x: {op: Exists} - feature: vendor.domain-c matchExpressions: vendor: {op: In, value: ["acme"]} will match the features in the NodeFeature object above and cause one more label to be created: feature.node.kubernetes.io/my-feature: "true" --- pkg/apis/nfd/v1alpha1/feature.go | 86 ++++++++++++++++++++++++++++++++ pkg/nfd-master/nfd-master.go | 16 +++--- 2 files changed, 95 insertions(+), 7 deletions(-) diff --git a/pkg/apis/nfd/v1alpha1/feature.go b/pkg/apis/nfd/v1alpha1/feature.go index 17948aaa9f..8615cdf64c 100644 --- a/pkg/apis/nfd/v1alpha1/feature.go +++ b/pkg/apis/nfd/v1alpha1/feature.go @@ -82,3 +82,89 @@ func (f *Features) Exists(name string) string { } return "" } + +// MergeInto merges two FeatureSpecs into one. Data in the input object takes +// precedence (overwrite) over data of the existing object we're merging into. +func (in *NodeFeatureSpec) MergeInto(out *NodeFeatureSpec) { + in.Features.MergeInto(&out.Features) + if in.Labels != nil { + if out.Labels == nil { + out.Labels = make(map[string]string, len(in.Labels)) + } + for key, val := range in.Labels { + out.Labels[key] = val + } + } +} + +// MergeInto merges two sets of features into one. Features from the input set +// take precedence (overwrite) features from the existing features of the set +// we're merging into. +func (in *Features) MergeInto(out *Features) { + if in.Flags != nil { + if out.Flags == nil { + out.Flags = make(map[string]FlagFeatureSet, len(in.Flags)) + } + for key, val := range in.Flags { + outVal := out.Flags[key] + val.MergeInto(&outVal) + out.Flags[key] = outVal + } + } + if in.Attributes != nil { + if out.Attributes == nil { + out.Attributes = make(map[string]AttributeFeatureSet, len(in.Attributes)) + } + for key, val := range in.Attributes { + outVal := out.Attributes[key] + val.MergeInto(&outVal) + out.Attributes[key] = outVal + } + } + if in.Instances != nil { + if out.Instances == nil { + out.Instances = make(map[string]InstanceFeatureSet, len(in.Instances)) + } + for key, val := range in.Instances { + outVal := out.Instances[key] + val.MergeInto(&outVal) + out.Instances[key] = outVal + } + } +} + +// MergeInto merges two sets of flag featues. +func (in *FlagFeatureSet) MergeInto(out *FlagFeatureSet) { + if in.Elements != nil { + if out.Elements == nil { + out.Elements = make(map[string]Nil, len(in.Elements)) + } + for key, val := range in.Elements { + out.Elements[key] = val + } + } +} + +// MergeInto merges two sets of attribute featues. +func (in *AttributeFeatureSet) MergeInto(out *AttributeFeatureSet) { + if in.Elements != nil { + if out.Elements == nil { + out.Elements = make(map[string]string, len(in.Elements)) + } + for key, val := range in.Elements { + out.Elements[key] = val + } + } +} + +// MergeInto merges two sets of instance featues. +func (in *InstanceFeatureSet) MergeInto(out *InstanceFeatureSet) { + if in.Elements != nil { + if out.Elements == nil { + out.Elements = make([]InstanceFeature, 0, len(in.Elements)) + } + for _, e := range in.Elements { + out.Elements = append(out.Elements, *e.DeepCopy()) + } + } +} diff --git a/pkg/nfd-master/nfd-master.go b/pkg/nfd-master/nfd-master.go index 7d1bdac3e2..a8abad9dba 100644 --- a/pkg/nfd-master/nfd-master.go +++ b/pkg/nfd-master/nfd-master.go @@ -516,12 +516,14 @@ func (m *nfdMaster) nfdAPIUpdateOneNode(nodeName string) error { // Merge in features // - // TODO: support multiple NodeFeature objects. There are two obvious options to implement this: - // 1. Merge features of all objects into one joint object - // 2. Change the rule api to support handle multiple objects - // Of these #2 would probably perform better with lot less data to copy. We - // could probably even get rid of the DeepCopy in this scenario. - features := objs[0].DeepCopy() + // NOTE: changing the rule api to support handle multiple objects instead + // of merging would probably perform better with lot less data to copy. + features := objs[0].Spec.DeepCopy() + for _, o := range objs[1:] { + o.Spec.MergeInto(features) + } + + utils.KlogDump(4, "Composite NodeFeatureSpec after merge:", " ", features) annotations := Annotations{} if objs[0].Namespace == m.namespace && objs[0].Name == nodeName { @@ -536,7 +538,7 @@ func (m *nfdMaster) nfdAPIUpdateOneNode(nodeName string) error { if err != nil { return err } - if err := m.refreshNodeFeatures(cli, nodeName, annotations, features.Spec.Labels, &features.Spec.Features); err != nil { + if err := m.refreshNodeFeatures(cli, nodeName, annotations, features.Labels, &features.Features); err != nil { return err } From 740e3af681e9f6cabaddbbd9a910f1d4cd233408 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Wed, 21 Sep 2022 21:19:57 +0300 Subject: [PATCH 51/87] nfd-master: implement ratelimiter for nfd api updates Implement a naive ratelimiter for node update events originating from the nfd API. We might get a ton of events in short interval. The simplest example is startup when we get a separate Add event for every NodeFeature and NodeFeatureRule object. Without rate limiting we run "update all nodes" separately for each NodeFeatureRule object, plus, we would run "update node X" separately for each NodeFeature object targeting node X. This is a huge amount of wasted work because in principle just running "update all nodes" once should be enough. --- pkg/nfd-master/nfd-master.go | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/pkg/nfd-master/nfd-master.go b/pkg/nfd-master/nfd-master.go index 7d1bdac3e2..75da6bd62e 100644 --- a/pkg/nfd-master/nfd-master.go +++ b/pkg/nfd-master/nfd-master.go @@ -259,16 +259,34 @@ func (m *nfdMaster) runGrpcServer(errChan chan<- error) { // nfdAPIUpdateHandler handles events from the nfd API controller. func (m *nfdMaster) nfdAPIUpdateHandler() { + updateAll := false + updateNodes := make(map[string]struct{}) + rateLimit := time.After(time.Second) for { select { case <-m.nfdController.updateAllNodesChan: - if err := m.nfdAPIUpdateAllNodes(); err != nil { - klog.Error(err) - } + updateAll = true case nodeName := <-m.nfdController.updateOneNodeChan: - if err := m.nfdAPIUpdateOneNode(nodeName); err != nil { - klog.Error(err) + updateNodes[nodeName] = struct{}{} + case <-rateLimit: + // Check what we need to do + // TODO: we might want to update multiple nodes in parallel + if updateAll { + if err := m.nfdAPIUpdateAllNodes(); err != nil { + klog.Error(err) + } + } else { + for nodeName := range updateNodes { + if err := m.nfdAPIUpdateOneNode(nodeName); err != nil { + klog.Error(err) + } + } } + + // Reset "work queue" and timer + updateAll = false + updateNodes = make(map[string]struct{}) + rateLimit = time.After(time.Second) } } } From ae8ba01c2c73da09425dd24cd3ece0e62a40ad87 Mon Sep 17 00:00:00 2001 From: Feruzjon Muyassarov Date: Wed, 14 Dec 2022 17:02:16 +0200 Subject: [PATCH 52/87] E2E: default seccompProfile to runtimeDefault for nfd worker Use RuntimeDefault seccomp profile in nfd worker and topology updater pod spec similar to nfd master. Signed-off-by: Feruzjon Muyassarov --- test/e2e/utils/pod/pod.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/e2e/utils/pod/pod.go b/test/e2e/utils/pod/pod.go index 674f38e6c5..b6396279bd 100644 --- a/test/e2e/utils/pod/pod.go +++ b/test/e2e/utils/pod/pod.go @@ -271,6 +271,9 @@ func nfdWorkerSpec(opts ...SpecOption) *corev1.PodSpec { RunAsNonRoot: &yes, ReadOnlyRootFilesystem: &yes, AllowPrivilegeEscalation: &no, + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, }, VolumeMounts: []corev1.VolumeMount{ { @@ -387,6 +390,9 @@ func NFDTopologyUpdaterSpec(kc utils.KubeletConfig, opts ...SpecOption) *corev1. RunAsUser: pointer.Int64(0), ReadOnlyRootFilesystem: pointer.Bool(true), AllowPrivilegeEscalation: pointer.Bool(false), + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, }, VolumeMounts: []corev1.VolumeMount{ { From 39813e7a73813757ad69c4d3045668ec4b9e7d03 Mon Sep 17 00:00:00 2001 From: Feruzjon Muyassarov Date: Wed, 14 Dec 2022 16:09:09 +0200 Subject: [PATCH 53/87] E2E: default kubeconfig location to ${HOME}/.kube/config Signed-off-by: Feruzjon Muyassarov --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3a116678ce..47a736d545 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ else endif HOSTMOUNT_PREFIX ?= / -KUBECONFIG ?= +KUBECONFIG ?= ${HOME}/.kube/config E2E_TEST_CONFIG ?= E2E_PULL_IF_NOT_PRESENT ?= false From 5d08f1cd7c3726ff77c3029d76d5888a11d002d1 Mon Sep 17 00:00:00 2001 From: Feruzjon Muyassarov Date: Wed, 14 Dec 2022 17:16:03 +0200 Subject: [PATCH 54/87] E2E:parameterize container image and tag Signed-off-by: Feruzjon Muyassarov --- test/e2e/node_feature_discovery.go | 15 ++++++++------- test/e2e/topology_updater.go | 4 ++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index e19144916a..4a55f54338 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -47,8 +47,9 @@ import ( ) var ( - dockerRepo = flag.String("nfd.repo", "gcr.io/k8s-staging-nfd/node-feature-discovery", "Docker repository to fetch image from") - dockerTag = flag.String("nfd.tag", "master", "Docker tag to use") + dockerRepo = flag.String("nfd.repo", "gcr.io/k8s-staging-nfd/node-feature-discovery", "Docker repository to fetch image from") + dockerTag = flag.String("nfd.tag", "master", "Docker tag to use") + dockerImage = fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag) ) // cleanupNode deletes all NFD-related metadata from the Node object, i.e. @@ -114,7 +115,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { // Launch nfd-master By("Creating nfd master pod and nfd-master service") - imageOpt := testpod.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)) + imageOpt := testpod.SpecWithContainerImage(dockerImage) masterPod = e2epod.NewPodClient(f).CreateSync(testpod.NFDMaster(imageOpt)) // Create nfd-master service @@ -159,7 +160,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { By("Creating a nfd worker pod") podSpecOpts := []testpod.SpecOption{ testpod.SpecWithRestartPolicy(corev1.RestartPolicyNever), - testpod.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)), + testpod.SpecWithContainerImage(dockerImage), testpod.SpecWithContainerExtraArgs("-oneshot", "-label-sources=fake"), } workerPod := testpod.NFDWorker(podSpecOpts...) @@ -208,7 +209,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { fConf := cfg.DefaultFeatures By("Creating nfd-worker daemonset") - podSpecOpts := []testpod.SpecOption{testpod.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag))} + podSpecOpts := []testpod.SpecOption{testpod.SpecWithContainerImage(dockerImage)} workerDS := testds.NFDWorker(podSpecOpts...) workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -339,7 +340,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { By("Creating nfd-worker daemonset with configmap mounted") podSpecOpts := []testpod.SpecOption{ - testpod.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)), + testpod.SpecWithContainerImage(dockerImage), testpod.SpecWithConfigMap(cm1.Name, filepath.Join(custom.Directory, "cm1")), testpod.SpecWithConfigMap(cm2.Name, filepath.Join(custom.Directory, "cm2")), } @@ -424,7 +425,7 @@ core: By("Creating nfd-worker daemonset") podSpecOpts := []testpod.SpecOption{ - testpod.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)), + testpod.SpecWithContainerImage(dockerImage), testpod.SpecWithConfigMap(cm.Name, "/etc/kubernetes/node-feature-discovery"), } workerDS := testds.NFDWorker(podSpecOpts...) diff --git a/test/e2e/topology_updater.go b/test/e2e/topology_updater.go index af69919948..8afd1399b2 100644 --- a/test/e2e/topology_updater.go +++ b/test/e2e/topology_updater.go @@ -115,7 +115,7 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { kcfg := cfg.GetKubeletConfig() By(fmt.Sprintf("Using config (%#v)", kcfg)) - podSpecOpts := []testpod.SpecOption{testpod.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag))} + podSpecOpts := []testpod.SpecOption{testpod.SpecWithContainerImage(dockerImage)} topologyUpdaterDaemonSet = testds.NFDTopologyUpdater(kcfg, podSpecOpts...) }) @@ -281,7 +281,7 @@ excludeList: By(fmt.Sprintf("Using config (%#v)", kcfg)) podSpecOpts := []testpod.SpecOption{ - testpod.SpecWithContainerImage(fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag)), + testpod.SpecWithContainerImage(dockerImage), testpod.SpecWithConfigMap(cm.Name, "/etc/kubernetes/node-feature-discovery"), } topologyUpdaterDaemonSet = testds.NFDTopologyUpdater(kcfg, podSpecOpts...) From 43910e6925e2bf1c050bbacf806f3ef3469f427d Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Wed, 14 Dec 2022 18:17:20 +0200 Subject: [PATCH 55/87] test/e2e: create CRDs once in the beginning of the tests Only generate CRDs once in the beginning of the test run. Use the "Ordered" option for the test container so that we can utilize ginkgo.BeforeAll to only do stuff once before the first test. Changing from unordered to ordered shouldn't make a big difference here. Add a cleanup function to remove stale NodeFeatureRule objects that are cluster-scoped and not deleted with the test namespace. --- test/e2e/node_feature_discovery.go | 69 ++++++++++++++++++------------ 1 file changed, 42 insertions(+), 27 deletions(-) diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index e19144916a..e3b7ff1e24 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -96,21 +96,58 @@ func cleanupNode(cs clientset.Interface) { } Expect(err).NotTo(HaveOccurred()) } + +} + +func cleanupCRDs(cli *nfdclient.Clientset) { + // Drop NodeFeatureRule objects + nfrs, err := cli.NfdV1alpha1().NodeFeatureRules().List(context.TODO(), metav1.ListOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Deleting NodeFeatureRule objects from the cluster") + for _, nfr := range nfrs.Items { + err = cli.NfdV1alpha1().NodeFeatureRules().Delete(context.TODO(), nfr.Name, metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + } } // Actual test suite -var _ = SIGDescribe("Node Feature Discovery", func() { +var _ = SIGDescribe("devel", func() { f := framework.NewDefaultFramework("node-feature-discovery") - Context("when deploying a single nfd-master pod", func() { - var masterPod *corev1.Pod + Context("when deploying a single nfd-master pod", Ordered, func() { + var ( + masterPod *corev1.Pod + crds []*apiextensionsv1.CustomResourceDefinition + extClient *extclient.Clientset + nfdClient *nfdclient.Clientset + ) + + BeforeAll(func() { + // Create clients for apiextensions and our CRD api + extClient = extclient.NewForConfigOrDie(f.ClientConfig()) + nfdClient = nfdclient.NewForConfigOrDie(f.ClientConfig()) + + By("Creating NFD CRDs") + var err error + crds, err = testutils.CreateNfdCRDs(extClient) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterAll(func() { + for _, crd := range crds { + err := extClient.ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), crd.Name, metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + } + }) BeforeEach(func() { err := testutils.ConfigureRBAC(f.ClientSet, f.Namespace.Name) Expect(err).NotTo(HaveOccurred()) - // Remove pre-existing stale annotations and labels + // Remove pre-existing stale annotations and labels etc and CRDs cleanupNode(f.ClientSet) + cleanupCRDs(nfdClient) // Launch nfd-master By("Creating nfd master pod and nfd-master service") @@ -141,6 +178,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { Expect(testutils.DeconfigureRBAC(f.ClientSet, f.Namespace.Name)).NotTo(HaveOccurred()) cleanupNode(f.ClientSet) + cleanupCRDs(nfdClient) }) // @@ -388,29 +426,6 @@ var _ = SIGDescribe("Node Feature Discovery", func() { // Test NodeFeatureRule // Context("and nfd-worker and NodeFeatureRules objects deployed", func() { - var extClient *extclient.Clientset - var nfdClient *nfdclient.Clientset - var crds []*apiextensionsv1.CustomResourceDefinition - - BeforeEach(func() { - // Create clients for apiextensions and our CRD api - extClient = extclient.NewForConfigOrDie(f.ClientConfig()) - nfdClient = nfdclient.NewForConfigOrDie(f.ClientConfig()) - - // Create CRDs - By("Creating NFD CRDs") - var err error - crds, err = testutils.CreateNfdCRDs(extClient) - Expect(err).NotTo(HaveOccurred()) - }) - - AfterEach(func() { - for _, crd := range crds { - err := extClient.ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), crd.Name, metav1.DeleteOptions{}) - Expect(err).NotTo(HaveOccurred()) - } - }) - It("custom labels from the NodeFeatureRule rules should be created", func() { By("Creating nfd-worker config") cm := testutils.NewConfigMap("nfd-worker-conf", "nfd-worker.conf", ` From b9c09e66747a16722b90c3d54cccbd5ecb01e921 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Wed, 14 Dec 2022 20:51:53 +0200 Subject: [PATCH 56/87] nfd-master: update all nodes at startup when NodeFeature API enabled We want to always update all nodes at startup. Without this patch we don't get any update event from the controller if no NodeFeature or NodeFeatureRule objects exist in the cluster. Thus all nodes would stay untouched whereas we really want to remove all labels from all nodes in this case. --- pkg/nfd-master/nfd-master.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/nfd-master/nfd-master.go b/pkg/nfd-master/nfd-master.go index 5589701fa6..c1b20bf1d5 100644 --- a/pkg/nfd-master/nfd-master.go +++ b/pkg/nfd-master/nfd-master.go @@ -259,7 +259,9 @@ func (m *nfdMaster) runGrpcServer(errChan chan<- error) { // nfdAPIUpdateHandler handles events from the nfd API controller. func (m *nfdMaster) nfdAPIUpdateHandler() { - updateAll := false + // We want to unconditionally update all nodes at startup if gRPC is + // disabled (i.e. NodeFeature API is enabled) + updateAll := m.args.EnableNodeFeatureApi updateNodes := make(map[string]struct{}) rateLimit := time.After(time.Second) for { From f04e1a940433daafafa214447d3292eb69ba12ec Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Wed, 14 Dec 2022 21:59:15 +0200 Subject: [PATCH 57/87] test/e2e: fix mistake in ginkgo focus Leftover from development that got accidentally merged in 43910e6925e2bf1c050bbacf806f3ef3469f427d --- test/e2e/node_feature_discovery.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index e3b7ff1e24..8986b4977e 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -112,7 +112,7 @@ func cleanupCRDs(cli *nfdclient.Clientset) { } // Actual test suite -var _ = SIGDescribe("devel", func() { +var _ = SIGDescribe("Node Feature Discovery", func() { f := framework.NewDefaultFramework("node-feature-discovery") Context("when deploying a single nfd-master pod", Ordered, func() { From 3209c14beaa5c84e392695a8929c25a6dd1498aa Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Thu, 1 Dec 2022 19:28:25 +0200 Subject: [PATCH 58/87] docs: document NodeFeature API Document the usage of the NodeFeature CRD API. Also re-organize the documentation a bit, moving the description of NodeFeatureRule controller from customization guide to nfd-master usage page. --- docs/deployment/helm.md | 2 +- docs/get-started/introduction.md | 8 +- .../reference/master-commandline-reference.md | 11 +- .../reference/worker-commandline-reference.md | 8 +- .../worker-configuration-reference.md | 4 +- docs/usage/custom-resources.md | 33 +++++ docs/usage/customization-guide.md | 115 +++++++++++++----- docs/usage/nfd-master.md | 51 +++++++- 8 files changed, 185 insertions(+), 47 deletions(-) diff --git a/docs/deployment/helm.md b/docs/deployment/helm.md index 03115988e9..49ab7052c6 100644 --- a/docs/deployment/helm.md +++ b/docs/deployment/helm.md @@ -97,7 +97,7 @@ We have introduced the following Chart parameters. | `fullnameOverride` | string | | Override a default fully qualified app name | | `tls.enable` | bool | false | Specifies whether to use TLS for communications between components | | `tls.certManager` | bool | false | If enabled, requires [cert-manager](https://cert-manager.io/docs/) to be installed and will automatically create the required TLS certificates | -| `enableNodeFeatureApi` | bool | false | Enable the NodeFeature CRD API for communicating node features. This will automatically disable the gRPC communication. +| `enableNodeFeatureApi` | bool | false | Enable the [NodeFeature](../usage/custom-resources#nodefeature) CRD API for communicating node features. This will automatically disable the gRPC communication. ### Master pod parameters diff --git a/docs/get-started/introduction.md b/docs/get-started/introduction.md index 3c6cac09a7..a4719def6e 100644 --- a/docs/get-started/introduction.md +++ b/docs/get-started/introduction.md @@ -106,9 +106,13 @@ only created on nodes running nfd-master. NFD takes use of some Kubernetes Custom Resources. -NFD-Master uses [NodeFeatureRule](../usage/custom-resources/nodefeaturerule)s +[NodeFeature](../usage/custom-resources#nodefeature)s (EXPERIMENTAL) +can be used for representing node features and requesting node labels to be +generated. + +NFD-Master uses [NodeFeatureRule](../usage/custom-resources#nodefeaturerule)s for custom labeling of nodes. NFD-Topology-Updater creates -[NodeResourceTopology](../usage/custom-resources/noderesourcetopology) objects +[NodeResourceTopology](../usage/custom-resources#noderesourcetopology) objects that describe the hardware topology of node resources. diff --git a/docs/reference/master-commandline-reference.md b/docs/reference/master-commandline-reference.md index 6bda05d505..0e9bbe7aa2 100644 --- a/docs/reference/master-commandline-reference.md +++ b/docs/reference/master-commandline-reference.md @@ -139,9 +139,9 @@ nfd-master -verify-node-name -ca-file=/opt/nfd/ca.crt \ ### -enable-nodefeature-api -The `-enable-nodefeature-api` flag enables the NodeFeature CRD API for -receiving feature requests. This will also automatically disable the gRPC -interface. +The `-enable-nodefeature-api` flag enables the +[NodeFeature](../usage/custom-resources#nodefeature) CRD API for receiving +feature requests. This will also automatically disable the gRPC interface. Default: false @@ -180,8 +180,9 @@ nfd-master -no-publish ### -crd-controller The `-crd-controller` flag specifies whether the NFD CRD API controller is -enabled or not. The controller is responsible for processing NodeFeature and -NodeFeatureRule objects. +enabled or not. The controller is responsible for processing +[NodeFeature](../usage/custom-resources#nodefeature) and +[NodeFeatureRule](../usage/custom-resources#nodefeaturerule) objects. Default: *true* diff --git a/docs/reference/worker-commandline-reference.md b/docs/reference/worker-commandline-reference.md index cf6651b8f2..55b90aa07e 100644 --- a/docs/reference/worker-commandline-reference.md +++ b/docs/reference/worker-commandline-reference.md @@ -125,8 +125,9 @@ nfd-worker -key-file=/opt/nfd/worker.key -cert-file=/opt/nfd/worker.crt -ca-file ### -kubeconfig The `-kubeconfig` flag specifies the kubeconfig to use for connecting to the -Kubernetes API server. It is only needed for manipulating NodeFeature -objects, and thus the flag only takes effect when +Kubernetes API server. It is only needed for manipulating +[NodeFeature](../usage/custom-resources#nodefeature) objects, and thus the flag +only takes effect when [`-enable-nodefeature-api`](#-enable-nodefeature-api)) is specified. An empty value (which is also the default) implies in-cluster kubeconfig. @@ -196,7 +197,8 @@ nfd-worker -label-sources=kernel,system,local ### -enable-nodefeature-api -The `-enable-nodefeature-api` flag enables the experimental NodeFeature CRD API +The `-enable-nodefeature-api` flag enables the experimental +[NodeFeature](../usage/custom-resources#nodefeature) CRD API for communicating with nfd-master. This will also automatically disable the gRPC communication to nfd-master. When enabled, nfd-worker will create per-node NodeFeature objects the contain all discovered node features and the set of diff --git a/docs/reference/worker-configuration-reference.md b/docs/reference/worker-configuration-reference.md index 48b13307cb..6680187c51 100644 --- a/docs/reference/worker-configuration-reference.md +++ b/docs/reference/worker-configuration-reference.md @@ -133,8 +133,8 @@ core: Setting `core.noPublish` to `true` disables all communication with the nfd-master and the Kubernetes API server. It is effectively a "dry-run" option. NFD-Worker runs feature detection normally, but no labeling requests are sent -to nfd-master and no NodeFeature objects are created or updated in the API -server. +to nfd-master and no [NodeFeature](../usage/custom-resources#nodefeature) +objects are created or updated in the API server. Note: Overridden by the [`-no-publish`](worker-commandline-reference#-no-publish) command line flag (if diff --git a/docs/usage/custom-resources.md b/docs/usage/custom-resources.md index 5d5e472f2f..deec64dec1 100644 --- a/docs/usage/custom-resources.md +++ b/docs/usage/custom-resources.md @@ -17,6 +17,39 @@ sort: 6 NFD uses some Kubernetes [custom resources][custom-resources]. +## NodeFeature + +**EXPERIMENTAL** +NodeFeatureRule is an NFD-specific custom resource for communicating node +features and node labeling requests. Support for NodeFeatureRule objects is +disabled by default. If enabled, nfd-master watches for NodeFeature objects, +labels nodes accordingly and uses the listed features as input when evaluating +[NodeFeatureRule](#nodefeaturerule)s. + +```yaml +apiVersion: nfd.k8s-sigs.io/v1alpha1 +kind: NodeFeature +metadata: + labels: + nfd.node.kubernetes.io/node-name: node-1 + name: node-1-vendor-features +spec: + features: + instances: + vendor.device: + elements: + - attributes: + model: "xpu-1" + memory: "4000" + type: "fast" + - attributes: + model: "xpu-2" + memory: "16000" + type: "slow" + labels: + vendor-xpu-present: "true" +``` + ## NodeFeatureRule NodeFeatureRule is an NFD-specific custom resource that is designed for diff --git a/docs/usage/customization-guide.md b/docs/usage/customization-guide.md index a9dba14eb5..a73c20d9b0 100644 --- a/docs/usage/customization-guide.md +++ b/docs/usage/customization-guide.md @@ -20,6 +20,9 @@ sort: 7 NFD provides multiple extension points for vendor and application specific labeling: +- [`NodeFeature`](#nodefeature-custom-resource) (EXPERIMENTAL) objects can be + used to communicate "raw" node features and node labeling requests to + nfd-master. - [`NodeFeatureRule`](#nodefeaturerule-custom-resource) objects provide a way to deploy custom labeling rules via the Kubernetes API - [`local`](#local-feature-source) feature source of nfd-worker creates @@ -27,6 +30,86 @@ labeling: - [`custom`](#custom-feature-source) feature source of nfd-worker creates labels based on user-specified rules +## NodeFeature custom resource + +**EXPERIMENTAL** +NodeFeature objects provide a way for 3rd party extensions to advertise custom +features, both as "raw" features that serve as input to +[NodeFeatureRule](#nodefeaturerule-custom-resource) objects and as feature +labels directly. + +Note that RBAC rules must be created for each extension for them to be able to +create and manipulate NodeFeature objects in their namespace. + +Support for NodeFeature CRD API is enabled with the `-enable-nodefeature-api` +command line flag. This flag must be specified for both nfd-master and +nfd-worker as it will disable the gRPC communication between them. + +### A NodeFeature example + +Consider the following referential example: + +```yaml +apiVersion: nfd.k8s-sigs.io/v1alpha1 +kind: NodeFeature +metadata: + labels: + nfd.node.kubernetes.io/node-name: node-1 + name: vendor-features-for-node-1 +spec: + # Features for NodeFeatureRule matching + features: + flags: + vendor.flags: + elements: + feature-x: {} + feature-y: {} + attributes: + vendor.config: + elements: + setting-a: "auto" + knob-b: "123" + instances: + vendor.devices: + elements: + - attributes: + model: "dev-1000" + vendor: "acme" + - attributes: + model: "dev-2000" + vendor: "acme" + # Labels to be created + labels: + vendor-feature.enabled: "true" +``` + +The object targets node named `node-1`. It lists two "flag type" features under +the `vendor.flags` domain, two "attribute type" features and uder the +`vendor.config` domain and two "instance type" features under the +`vendor.devices` domain. This features will not be directly affecting the node +labels but they will be used as input when be +[`NodeFeatureRule`](#nodefeaturerule-custom-resource) object are evaluated. + +In addition the example requests directly the +`feature.node.kubenernetes.io/vendor-feature.enabled=true` node label to be +created. + +The `nfd.node.kubernetes.io/node-name=` must be in place for each +NodeFeature objectt as NFD uses it to determine the node which it is targeting. + +### Feature types + +Features are divided into three different types: + +- **flag** features: a set of names without any associated values, e.g. CPUID + flags or loaded kernel modules +- **attribute** features: a set of names each of which has a single value + associated with it (essentially a map of key-value pairs), e.g. kernel config + flags or os release information +- **instance** features: a list of instances, each of which has multiple + attributes (key-value pairs of their own) associated with it, e.g. PCI or USB + devices + ## NodeFeatureRule custom resource `NodeFeatureRule` objects provide an easy way to create vendor or application @@ -125,21 +208,6 @@ spec: In this example, if the `my sample taint rule` rule is matched, `feature.node.kubernetes.io/pci-0300_1d0f.present=true:NoExecute` and `feature.node.kubernetes.io/cpu-cpuid.ADX:NoExecute` taints are set on the node. -### NodeFeatureRule controller - -NFD-Master acts as the controller for `NodeFeatureRule` objects. It applies these -rules on raw feature data received from nfd-worker instances and creates node -labels, accordingly. - -**NOTE** nfd-master is stateless and (re-)labelling only happens when a request -is received from nfd-worker. That is, in practice rules are evaluated and -labels for each node are created on intervals specified by the -[`core.sleepInterval`](../reference/worker-configuration-reference#coresleepinterval) -configuration option of nfd-worker instances. This means that modification or -creation of `NodeFeatureRule` objects does not instantly cause the node labels -to be updated. Instead, the changes only come visible in node labels as -nfd-worker instances send their labelling requests. - ## Local feature source NFD-Worker has a special feature source named `local` which is an integration @@ -523,24 +591,9 @@ true). ### Available features -#### Feature types - -Features are divided into three different types: - -- **flag** features: a set of names without any associated values, e.g. CPUID - flags or loaded kernel modules -- **attribute** features: a set of names each of which has a single value - associated with it (essentially a map of key-value pairs), e.g. kernel config - flags or os release information -- **instance** features: a list of instances, each of which has multiple - attributes (key-value pairs of their own) associated with it, e.g. PCI or USB - devices - -#### List of features - The following features are available for matching: -| Feature | Feature type | Elements | Value type | Description +| Feature | [Feature type](#feature-types) | Elements | Value type | Description | ---------------- | ------------ | -------- | ---------- | ----------- | **`cpu.cpuid`** | flag | | | Supported CPU capabilities | | | **``** | | CPUID flag is present diff --git a/docs/usage/nfd-master.md b/docs/usage/nfd-master.md index ac6020b333..6b1040253b 100644 --- a/docs/usage/nfd-master.md +++ b/docs/usage/nfd-master.md @@ -9,7 +9,52 @@ sort: 3 --- -NFD-Master runs as a deployment (with a replica count of 1), by default +NFD-Master is responsible for connecting to the Kubernetes API server and +updating node objects. More specifically, it modifies node labels, taints and +extended resources based on requests from nfd-workers and 3rd party extensions. + +## NodeFeature controller + +**EXPERIMENTAL** +Controller for [NodeFeature](custom-resources#nodefeature-custom-resource) +objects can be enabled with the +[`-enable-nodefeature-api`](../reference/master-commandline-reference#-enable-nodefeature-api) +command line flag. When enabled, features from NodeFeature objects are used as +the input for the [NodeFeatureRule](custom-resources#nodefeaturerule) +processing pipeline. In addition, any labels listed in the NodeFeature object +are created on the node (note the allowed +[label namespaces](customization-guide#node-labels) are controlled). + +> NOTE: NodeFeature API must also be enabled in nfd-worker with +> its [`-enable-nodefeature-api`](../reference/worker-commandline-reference#-enable-nodefeature-api) +> flag. + +## NodeFeatureRule controller + +NFD-Master acts as the controller for +[NodeFeatureRule](custom-resources#nodefeaturerule) objects. +It applies the rules specified in NodeFeatureRule objects on raw feature data +and creates node labels accordingly. The feature data used as the input can be +received from nfd-worker instances through the gRPC interface or from +[NodeFeature](custom-resources#nodefeature-custom-resource) objects. The latter +requires that the [NodeFeaure controller](#nodefeature-controller) has been +enabled. + +> NOTE: when gRPC is used for communicating the features (the default +> mechanism), (re-)labelling only happens when a request is received from +> nfd-worker. That is, in practice rules are evaluated and labels for each node +> are created on intervals specified by the +> [`core.sleepInterval`](../reference/worker-configuration-reference#coresleepinterval) +> configuration option of nfd-worker instances. This means that modification or +> creation of NodeFeatureRule objects does not instantly cause the node +> labels to be updated. Instead, the changes only come visible in node labels +> as nfd-worker instances send their labelling requests. This limitation is not +> present when gRPC interface is disabled +> and [NodeFeature](custom-resources#nodefeature-custom-resource) API is used. + +## Deployment notes + +NFD-Master runs as a deployment, by default it prefers running on the cluster's master nodes but will run on worker nodes if no master nodes are found. @@ -20,8 +65,8 @@ affinity to prevent masters from running on the same node. However note that inter-pod affinity is costly and is not recommended in bigger clusters. -NFD-Master listens for connections from nfd-worker(s) and connects to the -Kubernetes API server to add node labels advertised by them. +> NOTE: If the [NodeFeature controller](#nodefeature-controller) is enabled the +> replica count should be 1. If you have RBAC authorization enabled (as is the default e.g. with clusters initialized with kubeadm) you need to configure the appropriate ClusterRoles, From fd0b8124333db0d81e045b8f55d3efba6a5955d9 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Thu, 15 Dec 2022 16:46:22 +0200 Subject: [PATCH 59/87] test/e2e: drop host-usr-src mount Allows running e2e-tests in some more restricted environments. --- test/e2e/utils/pod/pod.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/test/e2e/utils/pod/pod.go b/test/e2e/utils/pod/pod.go index b6396279bd..4606792734 100644 --- a/test/e2e/utils/pod/pod.go +++ b/test/e2e/utils/pod/pod.go @@ -296,11 +296,6 @@ func nfdWorkerSpec(opts ...SpecOption) *corev1.PodSpec { MountPath: "/host-usr/lib", ReadOnly: true, }, - { - Name: "host-usr-src", - MountPath: "/host-usr/src", - ReadOnly: true, - }, }, }, }, @@ -343,15 +338,6 @@ func nfdWorkerSpec(opts ...SpecOption) *corev1.PodSpec { }, }, }, - { - Name: "host-usr-src", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/usr/src", - Type: newHostPathType(corev1.HostPathDirectory), - }, - }, - }, }, } From 7b2add434c8d382ca4e2cf34fe9753adad48ccb5 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Thu, 15 Dec 2022 16:40:20 +0200 Subject: [PATCH 60/87] test/e2e: drop pod security enforcement label from the test namespace Drop the pod-security.kubernetes.io/enforce label from the test namespace, i.e. remove pod security admission enforcement. NFD-worker uses restricted host mounts (/sys) etc so pod creation fails even in privileged mode if pod security admission enforcement is enabled. --- test/e2e/node_feature_discovery.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index 8986b4977e..47d275ccd6 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -37,6 +37,7 @@ import ( e2elog "k8s.io/kubernetes/test/e2e/framework" e2enetwork "k8s.io/kubernetes/test/e2e/framework/network" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + admissionapi "k8s.io/pod-security-admission/api" nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/pkg/apis/nfd/v1alpha1" nfdclient "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned" @@ -142,6 +143,14 @@ var _ = SIGDescribe("Node Feature Discovery", func() { }) BeforeEach(func() { + // Drop the pod security admission label as nfd-worker needs host mounts + if _, ok := f.Namespace.Labels[admissionapi.EnforceLevelLabel]; ok { + e2elog.Logf("Deleting %s label from the test namespace", admissionapi.EnforceLevelLabel) + delete(f.Namespace.Labels, admissionapi.EnforceLevelLabel) + _, err := f.ClientSet.CoreV1().Namespaces().Update(context.TODO(), f.Namespace, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + } + err := testutils.ConfigureRBAC(f.ClientSet, f.Namespace.Name) Expect(err).NotTo(HaveOccurred()) From 51f4c5447bb47dfff64b8ad7af71c766f3e17296 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Fri, 16 Dec 2022 22:10:36 +0200 Subject: [PATCH 61/87] test/e2e: don't expect control-plane nodes to be labeled Without this e2e tests may fail in clusters with control-plane-only nodes. --- test/e2e/node_feature_discovery.go | 43 ++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index 7b0fa5ef82..f2572313ae 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -33,6 +33,7 @@ import ( extclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" + taintutils "k8s.io/kubernetes/pkg/util/taints" "k8s.io/kubernetes/test/e2e/framework" e2elog "k8s.io/kubernetes/test/e2e/framework" e2enetwork "k8s.io/kubernetes/test/e2e/framework/network" @@ -333,17 +334,10 @@ var _ = SIGDescribe("Node Feature Discovery", func() { By("Getting a worker node") // We need a valid nodename for the configmap - nodeList, err := f.ClientSet.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + nodes, err := getNonControlPlaneNodes(f.ClientSet) Expect(err).NotTo(HaveOccurred()) - Expect(len(nodeList.Items)).ToNot(BeZero()) - targetNodeName := nodeList.Items[0].Name - for _, node := range nodeList.Items { - if _, ok := node.Labels["node-role.kubernetes.io/master"]; !ok { - targetNodeName = node.Name - break - } - } + targetNodeName := nodes[0].Name Expect(targetNodeName).ToNot(BeEmpty(), "No worker node found") // create a wildcard name as well for this node @@ -488,11 +482,11 @@ core: // waitForNfdNodeLabels waits for node to be labeled as expected. func waitForNfdNodeLabels(cli clientset.Interface, expected map[string]string) error { poll := func() error { - nodeList, err := cli.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + nodes, err := getNonControlPlaneNodes(cli) if err != nil { return err } - for _, node := range nodeList.Items { + for _, node := range nodes { labels := nfdLabels(node.Labels) if !cmp.Equal(expected, labels) { return fmt.Errorf("node %q labels do not match expected, diff (expected vs. received): %s", node.Name, cmp.Diff(expected, labels)) @@ -512,6 +506,33 @@ func waitForNfdNodeLabels(cli clientset.Interface, expected map[string]string) e return err } +// getNonControlPlaneNodes gets the nodes that are not tainted for exclusive control-plane usage +func getNonControlPlaneNodes(cli clientset.Interface) ([]corev1.Node, error) { + nodeList, err := cli.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return nil, err + } + if len(nodeList.Items) == 0 { + return nil, fmt.Errorf("no nodes found in the cluster") + } + + controlPlaneTaint := corev1.Taint{ + Effect: corev1.TaintEffectNoSchedule, + Key: "node-role.kubernetes.io/control-plane", + } + out := []corev1.Node{} + for _, node := range nodeList.Items { + if !taintutils.TaintExists(node.Spec.Taints, &controlPlaneTaint) { + out = append(out, node) + } + } + + if len(out) == 0 { + return nil, fmt.Errorf("no non-control-plane nodes found in the cluster") + } + return out, nil +} + // nfdLabels gets labels that are in the nfd label namespace. func nfdLabels(labels map[string]string) map[string]string { ret := map[string]string{} From 4c955ad72c462494f1c2cc45a44f858f2568fb2a Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Wed, 14 Dec 2022 20:50:49 +0200 Subject: [PATCH 62/87] nfd-master: update node if no NodeFeature objects are present Correctly handle the case where no NodeFeature objects exist for certain node (and NodeFeature API has been enabled with -enable-nodefeature-api). In this case all the labels should be removed. --- pkg/nfd-master/nfd-master.go | 41 +++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/pkg/nfd-master/nfd-master.go b/pkg/nfd-master/nfd-master.go index 5589701fa6..9e1b2480a4 100644 --- a/pkg/nfd-master/nfd-master.go +++ b/pkg/nfd-master/nfd-master.go @@ -502,10 +502,7 @@ func (m *nfdMaster) nfdAPIUpdateAllNodes() error { func (m *nfdMaster) nfdAPIUpdateOneNode(nodeName string) error { sel := labels.SelectorFromSet(labels.Set{nfdv1alpha1.NodeFeatureObjNodeNameLabel: nodeName}) objs, err := m.nfdController.featureLister.List(sel) - if len(objs) == 0 { - klog.Infof("no NodeFeature object exists for node %q, skipping...", nodeName) - return nil - } else if err != nil { + if err != nil { return fmt.Errorf("failed to get NodeFeature resources for node %q: %w", nodeName, err) } @@ -532,26 +529,32 @@ func (m *nfdMaster) nfdAPIUpdateOneNode(nodeName string) error { klog.V(1).Infof("processing node %q, initiated by NodeFeature API", nodeName) - // Merge in features - // - // NOTE: changing the rule api to support handle multiple objects instead - // of merging would probably perform better with lot less data to copy. - features := objs[0].Spec.DeepCopy() - for _, o := range objs[1:] { - o.Spec.MergeInto(features) - } + features := &nfdv1alpha1.NodeFeatureSpec{} + annotations := Annotations{} - utils.KlogDump(4, "Composite NodeFeatureSpec after merge:", " ", features) + if len(objs) > 0 { + // Merge in features + // + // NOTE: changing the rule api to support handle multiple objects instead + // of merging would probably perform better with lot less data to copy. + features = objs[0].Spec.DeepCopy() + for _, o := range objs[1:] { + o.Spec.MergeInto(features) + } - annotations := Annotations{} - if objs[0].Namespace == m.namespace && objs[0].Name == nodeName { - // This is the one created by nfd-worker - if v := objs[0].Annotations[nfdv1alpha1.WorkerVersionAnnotation]; v != "" { - annotations[nfdv1alpha1.WorkerVersionAnnotation] = v + utils.KlogDump(4, "Composite NodeFeatureSpec after merge:", " ", features) + + if objs[0].Namespace == m.namespace && objs[0].Name == nodeName { + // This is the one created by nfd-worker + if v := objs[0].Annotations[nfdv1alpha1.WorkerVersionAnnotation]; v != "" { + annotations[nfdv1alpha1.WorkerVersionAnnotation] = v + } } } - // Create labels et al + // Update node labels et al. This may also mean removing all NFD-owned + // labels (et al.), for example in the case no NodeFeature objects are + // present. cli, err := m.apihelper.GetClient() if err != nil { return err From 3e29f5c5df5914a724c2c8ca750cf7d5d16245c3 Mon Sep 17 00:00:00 2001 From: Feruzjon Muyassarov Date: Thu, 1 Dec 2022 01:21:21 +0200 Subject: [PATCH 63/87] Add nfd E2E tests for tainting feature Extend current E2E tests to check tainting feature of nfd implemented in https://github.com/kubernetes-sigs/node-feature-discovery/pull/910 Signed-off-by: Feruzjon Muyassarov --- test/e2e/data/nodefeaturerule-3-updated.yaml | 32 ++++ test/e2e/data/nodefeaturerule-3.yaml | 35 ++++ test/e2e/node_feature_discovery.go | 179 ++++++++++++++++++- test/e2e/utils/crd.go | 21 +++ test/e2e/utils/pod/pod.go | 8 + 5 files changed, 268 insertions(+), 7 deletions(-) create mode 100644 test/e2e/data/nodefeaturerule-3-updated.yaml create mode 100644 test/e2e/data/nodefeaturerule-3.yaml diff --git a/test/e2e/data/nodefeaturerule-3-updated.yaml b/test/e2e/data/nodefeaturerule-3-updated.yaml new file mode 100644 index 0000000000..8b17789788 --- /dev/null +++ b/test/e2e/data/nodefeaturerule-3-updated.yaml @@ -0,0 +1,32 @@ +apiVersion: nfd.k8s-sigs.io/v1alpha1 +kind: NodeFeatureRule +metadata: + name: e2e-test-3 +spec: + rules: + # Positive test expected to set the taints + - name: "e2e-taint-test-1" + taints: + - effect: PreferNoSchedule + key: "nfd.node.kubernetes.io/fake-special-node" + value: "exists" + - effect: NoExecute + key: "nfd.node.kubernetes.io/foo" + value: "true" + matchFeatures: + - feature: "fake.attribute" + matchExpressions: + "attr_1": {op: IsTrue} + "attr_2": {op: IsFalse} + + # Negative test not supposed to set the taints + - name: "e2e-taint-test-2" + taints: + - effect: PreferNoSchedule + key: "nfd.node.kubernetes.io/fake-cpu" + value: "true" + matchFeatures: + - feature: "fake.attribute" + matchExpressions: + "attr_1": {op: IsTrue} + "attr_2": {op: IsTrue} diff --git a/test/e2e/data/nodefeaturerule-3.yaml b/test/e2e/data/nodefeaturerule-3.yaml new file mode 100644 index 0000000000..b9701cb196 --- /dev/null +++ b/test/e2e/data/nodefeaturerule-3.yaml @@ -0,0 +1,35 @@ +apiVersion: nfd.k8s-sigs.io/v1alpha1 +kind: NodeFeatureRule +metadata: + name: e2e-test-3 +spec: + rules: + # Positive test expected to set the taints + - name: "e2e-taint-test-1" + taints: + - effect: PreferNoSchedule + key: "nfd.node.kubernetes.io/fake-special-node" + value: "exists" + - effect: NoExecute + key: "nfd.node.kubernetes.io/fake-dedicated-node" + value: "true" + - effect: "NoExecute" + key: "nfd.node.kubernetes.io/performance-optimized-node" + value: "true" + matchFeatures: + - feature: "fake.attribute" + matchExpressions: + "attr_1": {op: IsTrue} + "attr_2": {op: IsFalse} + + # Negative test not supposed to set the taints + - name: "e2e-taint-test-2" + taints: + - effect: PreferNoSchedule + key: "nfd.node.kubernetes.io/fake-special-cpu" + value: "true" + matchFeatures: + - feature: "fake.attribute" + matchExpressions: + "attr_1": {op: IsTrue} + "attr_2": {op: IsTrue} diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index f2572313ae..c4c97e33c4 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -49,11 +49,35 @@ import ( ) var ( - dockerRepo = flag.String("nfd.repo", "gcr.io/k8s-staging-nfd/node-feature-discovery", "Docker repository to fetch image from") - dockerTag = flag.String("nfd.tag", "master", "Docker tag to use") - dockerImage = fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag) + dockerRepo = flag.String("nfd.repo", "gcr.io/k8s-staging-nfd/node-feature-discovery", "Docker repository to fetch image from") + dockerTag = flag.String("nfd.tag", "master", "Docker tag to use") + dockerImage = fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag) + testTolerations = []corev1.Toleration{ + { + Key: "nfd.node.kubernetes.io/fake-special-node", + Value: "exists", + Effect: "NoExecute", + }, + { + Key: "nfd.node.kubernetes.io/fake-dedicated-node", + Value: "true", + Effect: "NoExecute", + }, + { + Key: "nfd.node.kubernetes.io/performance-optimized-node", + Value: "true", + Effect: "NoExecute", + }, + { + Key: "nfd.node.kubernetes.io/foo", + Value: "true", + Effect: "NoExecute", + }, + } ) +const TestTaintNs = "nfd.node.kubernetes.io" + // cleanupNode deletes all NFD-related metadata from the Node object, i.e. // labels and annotations func cleanupNode(cs clientset.Interface) { @@ -84,11 +108,22 @@ func cleanupNode(cs clientset.Interface) { } } + // Remove taints + for _, taint := range node.Spec.Taints { + if strings.HasPrefix(taint.Key, TestTaintNs) { + newTaints, removed := taintutils.DeleteTaint(node.Spec.Taints, &taint) + if removed { + node.Spec.Taints = newTaints + update = true + } + } + } + if !update { break } - By("Deleting NFD labels and annotations from node " + node.Name) + By("Deleting NFD labels, annotations and taints from node " + node.Name) _, err = cs.CoreV1().Nodes().Update(context.TODO(), node, metav1.UpdateOptions{}) if err != nil { time.Sleep(100 * time.Millisecond) @@ -162,8 +197,13 @@ var _ = SIGDescribe("Node Feature Discovery", func() { // Launch nfd-master By("Creating nfd master pod and nfd-master service") - imageOpt := testpod.SpecWithContainerImage(dockerImage) - masterPod = e2epod.NewPodClient(f).CreateSync(testpod.NFDMaster(imageOpt)) + + imageOpt := []testpod.SpecOption{ + testpod.SpecWithContainerImage(dockerImage), + testpod.SpecWithTolerations(testTolerations), + testpod.SpecWithContainerExtraArgs("-enable-taints"), + } + masterPod = e2epod.NewPodClient(f).CreateSync(testpod.NFDMaster(imageOpt...)) // Create nfd-master service nfdSvc, err := testutils.CreateService(f.ClientSet, f.Namespace.Name) @@ -210,6 +250,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { testpod.SpecWithRestartPolicy(corev1.RestartPolicyNever), testpod.SpecWithContainerImage(dockerImage), testpod.SpecWithContainerExtraArgs("-oneshot", "-label-sources=fake"), + testpod.SpecWithTolerations(testTolerations), } workerPod := testpod.NFDWorker(podSpecOpts...) workerPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), workerPod, metav1.CreateOptions{}) @@ -257,7 +298,10 @@ var _ = SIGDescribe("Node Feature Discovery", func() { fConf := cfg.DefaultFeatures By("Creating nfd-worker daemonset") - podSpecOpts := []testpod.SpecOption{testpod.SpecWithContainerImage(dockerImage)} + podSpecOpts := []testpod.SpecOption{ + testpod.SpecWithContainerImage(dockerImage), + testpod.SpecWithTolerations(testTolerations), + } workerDS := testds.NFDWorker(podSpecOpts...) workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -268,6 +312,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { By("Getting node objects") nodeList, err := f.ClientSet.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeList.Items)).ToNot(BeZero()) for _, node := range nodeList.Items { nodeConf := testutils.FindNodeConfig(cfg, node.Name) @@ -384,6 +429,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { testpod.SpecWithContainerImage(dockerImage), testpod.SpecWithConfigMap(cm1.Name, filepath.Join(custom.Directory, "cm1")), testpod.SpecWithConfigMap(cm2.Name, filepath.Join(custom.Directory, "cm2")), + testpod.SpecWithTolerations(testTolerations), } workerDS := testds.NFDWorker(podSpecOpts...) @@ -445,6 +491,7 @@ core: podSpecOpts := []testpod.SpecOption{ testpod.SpecWithContainerImage(dockerImage), testpod.SpecWithConfigMap(cm.Name, "/etc/kubernetes/node-feature-discovery"), + testpod.SpecWithTolerations(testTolerations), } workerDS := testds.NFDWorker(podSpecOpts...) workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) @@ -474,11 +521,87 @@ core: By("Verifying node labels from NodeFeatureRules #1 and #2") Expect(waitForNfdNodeLabels(f.ClientSet, expected)).NotTo(HaveOccurred()) + + // Add features from NodeFeatureRule #3 + By("Creating NodeFeatureRules #3") + Expect(testutils.CreateNodeFeatureRulesFromFile(nfdClient, "nodefeaturerule-3.yaml")).NotTo(HaveOccurred()) + + By("Verifying node taints and annotation from NodeFeatureRules #3") + expectedTaints := []corev1.Taint{ + { + Key: "nfd.node.kubernetes.io/fake-special-node", + Value: "exists", + Effect: "PreferNoSchedule", + }, + { + Key: "nfd.node.kubernetes.io/fake-dedicated-node", + Value: "true", + Effect: "NoExecute", + }, + { + Key: "nfd.node.kubernetes.io/performance-optimized-node", + Value: "true", + Effect: "NoExecute", + }, + } + expectedAnnotation := map[string]string{ + "nfd.node.kubernetes.io/taints": "nfd.node.kubernetes.io/fake-special-node=exists:PreferNoSchedule,nfd.node.kubernetes.io/fake-dedicated-node=true:NoExecute,nfd.node.kubernetes.io/performance-optimized-node=true:NoExecute"} + Expect(waitForNfdNodeTaints(f.ClientSet, expectedTaints)).NotTo(HaveOccurred()) + Expect(waitForNfdNodeAnnotations(f.ClientSet, expectedAnnotation)).NotTo(HaveOccurred()) + + By("Re-applying NodeFeatureRules #3 with updated taints") + Expect(testutils.UpdateNodeFeatureRulesFromFile(nfdClient, "nodefeaturerule-3-updated.yaml")).NotTo(HaveOccurred()) + expectedTaintsUpdated := []corev1.Taint{ + { + Key: "nfd.node.kubernetes.io/fake-special-node", + Value: "exists", + Effect: "PreferNoSchedule", + }, + { + Key: "nfd.node.kubernetes.io/foo", + Value: "true", + Effect: "NoExecute", + }, + } + expectedAnnotationUpdated := map[string]string{ + "nfd.node.kubernetes.io/taints": "nfd.node.kubernetes.io/fake-special-node=exists:PreferNoSchedule,nfd.node.kubernetes.io/foo=true:NoExecute"} + + By("Verifying updated node taints and annotation from NodeFeatureRules #3") + Expect(waitForNfdNodeTaints(f.ClientSet, expectedTaintsUpdated)).NotTo(HaveOccurred()) + Expect(waitForNfdNodeAnnotations(f.ClientSet, expectedAnnotationUpdated)).NotTo(HaveOccurred()) }) }) }) }) +// waitForNfdNodeAnnotations waits for node to be annotated as expected. +func waitForNfdNodeAnnotations(cli clientset.Interface, expected map[string]string) error { + poll := func() error { + nodes, err := getNonControlPlaneNodes(cli) + if err != nil { + return err + } + for _, node := range nodes { + for k, v := range expected { + if diff := cmp.Diff(v, node.Annotations[k]); diff != "" { + return fmt.Errorf("node %q annotation does not match expected, diff (expected vs. received): %s", node.Name, diff) + } + } + } + return nil + } + + // Simple and stupid re-try loop + var err error + for retry := 0; retry < 3; retry++ { + if err = poll(); err == nil { + return nil + } + time.Sleep(2 * time.Second) + } + return err +} + // waitForNfdNodeLabels waits for node to be labeled as expected. func waitForNfdNodeLabels(cli clientset.Interface, expected map[string]string) error { poll := func() error { @@ -506,6 +629,48 @@ func waitForNfdNodeLabels(cli clientset.Interface, expected map[string]string) e return err } +// waitForNfdNodeTaints waits for node to be tainted as expected. +func waitForNfdNodeTaints(cli clientset.Interface, expected []corev1.Taint) error { + poll := func() error { + nodes, err := getNonControlPlaneNodes(cli) + if err != nil { + return err + } + for _, node := range nodes { + taints := nfdTaints(node.Spec.Taints) + if err != nil { + return fmt.Errorf("failed to fetch nfd owned taints for node: %s", node.Name) + } + if !cmp.Equal(expected, taints) { + return fmt.Errorf("node %q taints do not match expected, diff (expected vs. received): %s", node.Name, cmp.Diff(expected, taints)) + } + } + return nil + } + + // Simple and stupid re-try loop + var err error + for retry := 0; retry < 3; retry++ { + if err = poll(); err == nil { + return nil + } + time.Sleep(10 * time.Second) + } + return err +} + +// nfdTaints returns taints that are owned by the nfd. +func nfdTaints(taints []corev1.Taint) []corev1.Taint { + nfdTaints := []corev1.Taint{} + for _, taint := range taints { + if strings.HasPrefix(taint.Key, TestTaintNs) { + nfdTaints = append(nfdTaints, taint) + } + } + + return nfdTaints +} + // getNonControlPlaneNodes gets the nodes that are not tainted for exclusive control-plane usage func getNonControlPlaneNodes(cli clientset.Interface) ([]corev1.Node, error) { nodeList, err := cli.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) diff --git a/test/e2e/utils/crd.go b/test/e2e/utils/crd.go index 76ea53c00d..6d95d90668 100644 --- a/test/e2e/utils/crd.go +++ b/test/e2e/utils/crd.go @@ -74,6 +74,27 @@ func CreateNodeFeatureRulesFromFile(cli nfdclientset.Interface, filename string) return nil } +// UpdateNodeFeatureRulesFromFile updates existing NodeFeatureRule object from a given file located under test data directory. +func UpdateNodeFeatureRulesFromFile(cli nfdclientset.Interface, filename string) error { + objs, err := nodeFeatureRulesFromFile(filepath.Join(packagePath, "..", "data", filename)) + if err != nil { + return err + } + + for _, obj := range objs { + var nfr *nfdv1alpha1.NodeFeatureRule + if nfr, err = cli.NfdV1alpha1().NodeFeatureRules().Get(context.TODO(), obj.Name, metav1.GetOptions{}); err != nil { + return fmt.Errorf("failed to get NodeFeatureRule %w", err) + } + + obj.SetResourceVersion(nfr.GetResourceVersion()) + if _, err = cli.NfdV1alpha1().NodeFeatureRules().Update(context.TODO(), obj, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("failed to update NodeFeatureRule %w", err) + } + } + return nil +} + func apiObjsFromFile(path string, decoder apiruntime.Decoder) ([]apiruntime.Object, error) { data, err := os.ReadFile(path) if err != nil { diff --git a/test/e2e/utils/pod/pod.go b/test/e2e/utils/pod/pod.go index b6396279bd..9cea5630fb 100644 --- a/test/e2e/utils/pod/pod.go +++ b/test/e2e/utils/pod/pod.go @@ -219,6 +219,14 @@ func SpecWithMasterNodeSelector(args ...string) SpecOption { } } +// SpecWithTolerations returns a SpecOption that modifies the pod to +// be run on a node with NodeFeatureRule taints. +func SpecWithTolerations(tolerations []corev1.Toleration) SpecOption { + return func(spec *corev1.PodSpec) { + spec.Tolerations = append(spec.Tolerations, tolerations...) + } +} + // SpecWithConfigMap returns a SpecOption that mounts a configmap to the first container. func SpecWithConfigMap(name, mountPath string) SpecOption { return func(spec *corev1.PodSpec) { From 958db566805e992ba9f0f5205a30fe11b2dc481d Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Wed, 14 Dec 2022 16:06:04 +0200 Subject: [PATCH 64/87] test/e2e: isolate tests into a separate function Preparation for running the same tests with NodeFeature API enabled (instead of gRPC). --- test/e2e/node_feature_discovery.go | 680 +++++++++++++++-------------- 1 file changed, 344 insertions(+), 336 deletions(-) diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index c4c97e33c4..bb6075bf4f 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -153,263 +153,264 @@ func cleanupCRDs(cli *nfdclient.Clientset) { var _ = SIGDescribe("Node Feature Discovery", func() { f := framework.NewDefaultFramework("node-feature-discovery") - Context("when deploying a single nfd-master pod", Ordered, func() { - var ( - masterPod *corev1.Pod - crds []*apiextensionsv1.CustomResourceDefinition - extClient *extclient.Clientset - nfdClient *nfdclient.Clientset - ) - - BeforeAll(func() { - // Create clients for apiextensions and our CRD api - extClient = extclient.NewForConfigOrDie(f.ClientConfig()) - nfdClient = nfdclient.NewForConfigOrDie(f.ClientConfig()) - - By("Creating NFD CRDs") - var err error - crds, err = testutils.CreateNfdCRDs(extClient) - Expect(err).NotTo(HaveOccurred()) - }) - - AfterAll(func() { - for _, crd := range crds { - err := extClient.ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), crd.Name, metav1.DeleteOptions{}) - Expect(err).NotTo(HaveOccurred()) - } - }) - - BeforeEach(func() { - // Drop the pod security admission label as nfd-worker needs host mounts - if _, ok := f.Namespace.Labels[admissionapi.EnforceLevelLabel]; ok { - e2elog.Logf("Deleting %s label from the test namespace", admissionapi.EnforceLevelLabel) - delete(f.Namespace.Labels, admissionapi.EnforceLevelLabel) - _, err := f.ClientSet.CoreV1().Namespaces().Update(context.TODO(), f.Namespace, metav1.UpdateOptions{}) + nfdTestSuite := func(useNodeFeatureApi bool) { + Context("when deploying a single nfd-master pod", Ordered, func() { + var ( + masterPod *corev1.Pod + crds []*apiextensionsv1.CustomResourceDefinition + extClient *extclient.Clientset + nfdClient *nfdclient.Clientset + ) + + BeforeAll(func() { + // Create clients for apiextensions and our CRD api + extClient = extclient.NewForConfigOrDie(f.ClientConfig()) + nfdClient = nfdclient.NewForConfigOrDie(f.ClientConfig()) + + By("Creating NFD CRDs") + var err error + crds, err = testutils.CreateNfdCRDs(extClient) Expect(err).NotTo(HaveOccurred()) - } - - err := testutils.ConfigureRBAC(f.ClientSet, f.Namespace.Name) - Expect(err).NotTo(HaveOccurred()) - - // Remove pre-existing stale annotations and labels etc and CRDs - cleanupNode(f.ClientSet) - cleanupCRDs(nfdClient) - - // Launch nfd-master - By("Creating nfd master pod and nfd-master service") - - imageOpt := []testpod.SpecOption{ - testpod.SpecWithContainerImage(dockerImage), - testpod.SpecWithTolerations(testTolerations), - testpod.SpecWithContainerExtraArgs("-enable-taints"), - } - masterPod = e2epod.NewPodClient(f).CreateSync(testpod.NFDMaster(imageOpt...)) - - // Create nfd-master service - nfdSvc, err := testutils.CreateService(f.ClientSet, f.Namespace.Name) - Expect(err).NotTo(HaveOccurred()) - - By("Waiting for the nfd-master pod to be running") - Expect(e2epod.WaitTimeoutForPodRunningInNamespace(f.ClientSet, masterPod.Name, masterPod.Namespace, time.Minute)).NotTo(HaveOccurred()) - - By("Verifying the node where nfd-master is running") - // Get updated masterPod object (we want to know where it was scheduled) - masterPod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(context.TODO(), masterPod.ObjectMeta.Name, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - // Node running nfd-master should have master version annotation - masterPodNode, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), masterPod.Spec.NodeName, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - Expect(masterPodNode.Annotations).To(HaveKey(nfdv1alpha1.AnnotationNs + "/master.version")) + }) - By("Waiting for the nfd-master service to be up") - Expect(e2enetwork.WaitForService(f.ClientSet, f.Namespace.Name, nfdSvc.ObjectMeta.Name, true, time.Second, 10*time.Second)).NotTo(HaveOccurred()) - }) + AfterAll(func() { + for _, crd := range crds { + err := extClient.ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), crd.Name, metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + } + }) - AfterEach(func() { - Expect(testutils.DeconfigureRBAC(f.ClientSet, f.Namespace.Name)).NotTo(HaveOccurred()) + BeforeEach(func() { + // Drop the pod security admission label as nfd-worker needs host mounts + if _, ok := f.Namespace.Labels[admissionapi.EnforceLevelLabel]; ok { + e2elog.Logf("Deleting %s label from the test namespace", admissionapi.EnforceLevelLabel) + delete(f.Namespace.Labels, admissionapi.EnforceLevelLabel) + _, err := f.ClientSet.CoreV1().Namespaces().Update(context.TODO(), f.Namespace, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + } - cleanupNode(f.ClientSet) - cleanupCRDs(nfdClient) - }) + err := testutils.ConfigureRBAC(f.ClientSet, f.Namespace.Name) + Expect(err).NotTo(HaveOccurred()) - // - // Simple test with only the fake source enabled - // - Context("and a single worker pod with fake source enabled", func() { - It("it should decorate the node with the fake feature labels", func() { + // Remove pre-existing stale annotations and labels etc and CRDs + cleanupNode(f.ClientSet) + cleanupCRDs(nfdClient) - fakeFeatureLabels := map[string]string{ - nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature1": "true", - nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature2": "true", - nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature3": "true", - } + // Launch nfd-master + By("Creating nfd master pod and nfd-master service") - // Launch nfd-worker - By("Creating a nfd worker pod") - podSpecOpts := []testpod.SpecOption{ - testpod.SpecWithRestartPolicy(corev1.RestartPolicyNever), + imageOpt := []testpod.SpecOption{ testpod.SpecWithContainerImage(dockerImage), - testpod.SpecWithContainerExtraArgs("-oneshot", "-label-sources=fake"), testpod.SpecWithTolerations(testTolerations), + testpod.SpecWithContainerExtraArgs("-enable-taints"), } - workerPod := testpod.NFDWorker(podSpecOpts...) - workerPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), workerPod, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) + masterPod = e2epod.NewPodClient(f).CreateSync(testpod.NFDMaster(imageOpt...)) - By("Waiting for the nfd-worker pod to succeed") - Expect(e2epod.WaitForPodSuccessInNamespace(f.ClientSet, workerPod.ObjectMeta.Name, f.Namespace.Name)).NotTo(HaveOccurred()) - workerPod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(context.TODO(), workerPod.ObjectMeta.Name, metav1.GetOptions{}) + // Create nfd-master service + nfdSvc, err := testutils.CreateService(f.ClientSet, f.Namespace.Name) Expect(err).NotTo(HaveOccurred()) - By(fmt.Sprintf("Making sure '%s' was decorated with the fake feature labels", workerPod.Spec.NodeName)) - node, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), workerPod.Spec.NodeName, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - for k, v := range fakeFeatureLabels { - Expect(node.Labels[k]).To(Equal(v)) - } + By("Waiting for the nfd-master pod to be running") + Expect(e2epod.WaitTimeoutForPodRunningInNamespace(f.ClientSet, masterPod.Name, masterPod.Namespace, time.Minute)).NotTo(HaveOccurred()) - // Check that there are no unexpected NFD labels - for k := range node.Labels { - if strings.HasPrefix(k, nfdv1alpha1.FeatureLabelNs) { - Expect(fakeFeatureLabels).Should(HaveKey(k)) - } - } - - By("Deleting the node-feature-discovery worker pod") - err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(context.TODO(), workerPod.ObjectMeta.Name, metav1.DeleteOptions{}) + By("Verifying the node where nfd-master is running") + // Get updated masterPod object (we want to know where it was scheduled) + masterPod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(context.TODO(), masterPod.ObjectMeta.Name, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) - }) - }) - - // - // More comprehensive test when --e2e-node-config is enabled - // - Context("and nfd-workers as a daemonset with default sources enabled", func() { - It("the node labels and annotations listed in the e2e config should be present", func() { - cfg, err := testutils.GetConfig() - Expect(err).ToNot(HaveOccurred()) + // Node running nfd-master should have master version annotation + masterPodNode, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), masterPod.Spec.NodeName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(masterPodNode.Annotations).To(HaveKey(nfdv1alpha1.AnnotationNs + "/master.version")) - if cfg == nil { - Skip("no e2e-config was specified") - } - if cfg.DefaultFeatures == nil { - Skip("no 'defaultFeatures' specified in e2e-config") - } - fConf := cfg.DefaultFeatures + By("Waiting for the nfd-master service to be up") + Expect(e2enetwork.WaitForService(f.ClientSet, f.Namespace.Name, nfdSvc.ObjectMeta.Name, true, time.Second, 10*time.Second)).NotTo(HaveOccurred()) + }) - By("Creating nfd-worker daemonset") - podSpecOpts := []testpod.SpecOption{ - testpod.SpecWithContainerImage(dockerImage), - testpod.SpecWithTolerations(testTolerations), - } - workerDS := testds.NFDWorker(podSpecOpts...) - workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) + AfterEach(func() { + Expect(testutils.DeconfigureRBAC(f.ClientSet, f.Namespace.Name)).NotTo(HaveOccurred()) - By("Waiting for daemonset pods to be ready") - Expect(testpod.WaitForReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred()) + cleanupNode(f.ClientSet) + cleanupCRDs(nfdClient) + }) - By("Getting node objects") - nodeList, err := f.ClientSet.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) - Expect(err).NotTo(HaveOccurred()) - Expect(len(nodeList.Items)).ToNot(BeZero()) + // + // Simple test with only the fake source enabled + // + Context("and a single worker pod with fake source enabled", func() { + It("it should decorate the node with the fake feature labels", func() { - for _, node := range nodeList.Items { - nodeConf := testutils.FindNodeConfig(cfg, node.Name) - if nodeConf == nil { - e2elog.Logf("node %q has no matching rule in e2e-config, skipping...", node.Name) - continue + fakeFeatureLabels := map[string]string{ + nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature1": "true", + nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature2": "true", + nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature3": "true", } - // Check labels - e2elog.Logf("verifying labels of node %q...", node.Name) - for k, v := range nodeConf.ExpectedLabelValues { - Expect(node.Labels).To(HaveKeyWithValue(k, v)) + // Launch nfd-worker + By("Creating a nfd worker pod") + podSpecOpts := []testpod.SpecOption{ + testpod.SpecWithRestartPolicy(corev1.RestartPolicyNever), + testpod.SpecWithContainerImage(dockerImage), + testpod.SpecWithContainerExtraArgs("-oneshot", "-label-sources=fake"), + testpod.SpecWithTolerations(testTolerations), } - for k := range nodeConf.ExpectedLabelKeys { - Expect(node.Labels).To(HaveKey(k)) + workerPod := testpod.NFDWorker(podSpecOpts...) + workerPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), workerPod, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for the nfd-worker pod to succeed") + Expect(e2epod.WaitForPodSuccessInNamespace(f.ClientSet, workerPod.ObjectMeta.Name, f.Namespace.Name)).NotTo(HaveOccurred()) + workerPod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(context.TODO(), workerPod.ObjectMeta.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By(fmt.Sprintf("Making sure '%s' was decorated with the fake feature labels", workerPod.Spec.NodeName)) + node, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), workerPod.Spec.NodeName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + for k, v := range fakeFeatureLabels { + Expect(node.Labels[k]).To(Equal(v)) } + + // Check that there are no unexpected NFD labels for k := range node.Labels { if strings.HasPrefix(k, nfdv1alpha1.FeatureLabelNs) { - if _, ok := nodeConf.ExpectedLabelValues[k]; ok { - continue - } - if _, ok := nodeConf.ExpectedLabelKeys[k]; ok { - continue - } - // Ignore if the label key was not whitelisted - Expect(fConf.LabelWhitelist).NotTo(HaveKey(k)) + Expect(fakeFeatureLabels).Should(HaveKey(k)) } } - // Check annotations - e2elog.Logf("verifying annotations of node %q...", node.Name) - for k, v := range nodeConf.ExpectedAnnotationValues { - Expect(node.Annotations).To(HaveKeyWithValue(k, v)) + By("Deleting the node-feature-discovery worker pod") + err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(context.TODO(), workerPod.ObjectMeta.Name, metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + // + // More comprehensive test when --e2e-node-config is enabled + // + Context("and nfd-workers as a daemonset with default sources enabled", func() { + It("the node labels and annotations listed in the e2e config should be present", func() { + cfg, err := testutils.GetConfig() + Expect(err).ToNot(HaveOccurred()) + + if cfg == nil { + Skip("no e2e-config was specified") } - for k := range nodeConf.ExpectedAnnotationKeys { - Expect(node.Annotations).To(HaveKey(k)) + if cfg.DefaultFeatures == nil { + Skip("no 'defaultFeatures' specified in e2e-config") } - for k := range node.Annotations { - if strings.HasPrefix(k, nfdv1alpha1.AnnotationNs) { - if _, ok := nodeConf.ExpectedAnnotationValues[k]; ok { - continue + fConf := cfg.DefaultFeatures + + By("Creating nfd-worker daemonset") + podSpecOpts := []testpod.SpecOption{ + testpod.SpecWithContainerImage(dockerImage), + testpod.SpecWithTolerations(testTolerations), + } + workerDS := testds.NFDWorker(podSpecOpts...) + workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for daemonset pods to be ready") + Expect(testpod.WaitForReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred()) + + By("Getting node objects") + nodeList, err := f.ClientSet.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(len(nodeList.Items)).ToNot(BeZero()) + + for _, node := range nodeList.Items { + nodeConf := testutils.FindNodeConfig(cfg, node.Name) + if nodeConf == nil { + e2elog.Logf("node %q has no matching rule in e2e-config, skipping...", node.Name) + continue + } + + // Check labels + e2elog.Logf("verifying labels of node %q...", node.Name) + for k, v := range nodeConf.ExpectedLabelValues { + Expect(node.Labels).To(HaveKeyWithValue(k, v)) + } + for k := range nodeConf.ExpectedLabelKeys { + Expect(node.Labels).To(HaveKey(k)) + } + for k := range node.Labels { + if strings.HasPrefix(k, nfdv1alpha1.FeatureLabelNs) { + if _, ok := nodeConf.ExpectedLabelValues[k]; ok { + continue + } + if _, ok := nodeConf.ExpectedLabelKeys[k]; ok { + continue + } + // Ignore if the label key was not whitelisted + Expect(fConf.LabelWhitelist).NotTo(HaveKey(k)) } - if _, ok := nodeConf.ExpectedAnnotationKeys[k]; ok { - continue + } + + // Check annotations + e2elog.Logf("verifying annotations of node %q...", node.Name) + for k, v := range nodeConf.ExpectedAnnotationValues { + Expect(node.Annotations).To(HaveKeyWithValue(k, v)) + } + for k := range nodeConf.ExpectedAnnotationKeys { + Expect(node.Annotations).To(HaveKey(k)) + } + for k := range node.Annotations { + if strings.HasPrefix(k, nfdv1alpha1.AnnotationNs) { + if _, ok := nodeConf.ExpectedAnnotationValues[k]; ok { + continue + } + if _, ok := nodeConf.ExpectedAnnotationKeys[k]; ok { + continue + } + // Ignore if the annotation was not whitelisted + Expect(fConf.AnnotationWhitelist).NotTo(HaveKey(k)) } - // Ignore if the annotation was not whitelisted - Expect(fConf.AnnotationWhitelist).NotTo(HaveKey(k)) } - } - } + } - By("Deleting nfd-worker daemonset") - err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Delete(context.TODO(), workerDS.ObjectMeta.Name, metav1.DeleteOptions{}) - Expect(err).NotTo(HaveOccurred()) + By("Deleting nfd-worker daemonset") + err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Delete(context.TODO(), workerDS.ObjectMeta.Name, metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + }) }) - }) - // - // Test custom nodename source configured in 2 additional ConfigMaps - // - Context("and nfd-workers as a daemonset with 2 additional configmaps for the custom source configured", func() { - It("the nodename matching features listed in the configmaps should be present", func() { - By("Getting a worker node") + // + // Test custom nodename source configured in 2 additional ConfigMaps + // + Context("and nfd-workers as a daemonset with 2 additional configmaps for the custom source configured", func() { + It("the nodename matching features listed in the configmaps should be present", func() { + By("Getting a worker node") - // We need a valid nodename for the configmap - nodes, err := getNonControlPlaneNodes(f.ClientSet) - Expect(err).NotTo(HaveOccurred()) + // We need a valid nodename for the configmap + nodes, err := getNonControlPlaneNodes(f.ClientSet) + Expect(err).NotTo(HaveOccurred()) - targetNodeName := nodes[0].Name - Expect(targetNodeName).ToNot(BeEmpty(), "No worker node found") + targetNodeName := nodes[0].Name + Expect(targetNodeName).ToNot(BeEmpty(), "No worker node found") - // create a wildcard name as well for this node - targetNodeNameWildcard := fmt.Sprintf("%s.*%s", targetNodeName[:2], targetNodeName[4:]) + // create a wildcard name as well for this node + targetNodeNameWildcard := fmt.Sprintf("%s.*%s", targetNodeName[:2], targetNodeName[4:]) - By("Creating the configmaps") - targetLabelName := "nodename-test" - targetLabelValue := "true" + By("Creating the configmaps") + targetLabelName := "nodename-test" + targetLabelValue := "true" - targetLabelNameWildcard := "nodename-test-wildcard" - targetLabelValueWildcard := "customValue" + targetLabelNameWildcard := "nodename-test-wildcard" + targetLabelValueWildcard := "customValue" - targetLabelNameNegative := "nodename-test-negative" + targetLabelNameNegative := "nodename-test-negative" - // create 2 configmaps - data1 := ` + // create 2 configmaps + data1 := ` - name: ` + targetLabelName + ` matchOn: # default value is true - nodename: - ` + targetNodeName - cm1 := testutils.NewConfigMap("custom-config-extra-1", "custom.conf", data1) - cm1, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(context.TODO(), cm1, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) + cm1 := testutils.NewConfigMap("custom-config-extra-1", "custom.conf", data1) + cm1, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(context.TODO(), cm1, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) - data2 := ` + data2 := ` - name: ` + targetLabelNameWildcard + ` value: ` + targetLabelValueWildcard + ` matchOn: @@ -420,157 +421,164 @@ var _ = SIGDescribe("Node Feature Discovery", func() { - nodename: - "thisNameShouldNeverMatch"` - cm2 := testutils.NewConfigMap("custom-config-extra-2", "custom.conf", data2) - cm2, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(context.TODO(), cm2, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) + cm2 := testutils.NewConfigMap("custom-config-extra-2", "custom.conf", data2) + cm2, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(context.TODO(), cm2, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) - By("Creating nfd-worker daemonset with configmap mounted") - podSpecOpts := []testpod.SpecOption{ - testpod.SpecWithContainerImage(dockerImage), - testpod.SpecWithConfigMap(cm1.Name, filepath.Join(custom.Directory, "cm1")), - testpod.SpecWithConfigMap(cm2.Name, filepath.Join(custom.Directory, "cm2")), - testpod.SpecWithTolerations(testTolerations), - } - workerDS := testds.NFDWorker(podSpecOpts...) + By("Creating nfd-worker daemonset with configmap mounted") + podSpecOpts := []testpod.SpecOption{ + testpod.SpecWithContainerImage(dockerImage), + testpod.SpecWithConfigMap(cm1.Name, filepath.Join(custom.Directory, "cm1")), + testpod.SpecWithConfigMap(cm2.Name, filepath.Join(custom.Directory, "cm2")), + testpod.SpecWithTolerations(testTolerations), + } + workerDS := testds.NFDWorker(podSpecOpts...) - workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) + workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) - By("Waiting for daemonset pods to be ready") - Expect(testpod.WaitForReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred()) + By("Waiting for daemonset pods to be ready") + Expect(testpod.WaitForReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred()) - By("Getting target node and checking labels") - targetNode, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), targetNodeName, metav1.GetOptions{}) - Expect(err).ToNot(HaveOccurred()) + By("Getting target node and checking labels") + targetNode, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), targetNodeName, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) - labelFound := false - labelWildcardFound := false - labelNegativeFound := false - for k := range targetNode.Labels { - if strings.Contains(k, targetLabelName) { - if targetNode.Labels[k] == targetLabelValue { - labelFound = true + labelFound := false + labelWildcardFound := false + labelNegativeFound := false + for k := range targetNode.Labels { + if strings.Contains(k, targetLabelName) { + if targetNode.Labels[k] == targetLabelValue { + labelFound = true + } } - } - if strings.Contains(k, targetLabelNameWildcard) { - if targetNode.Labels[k] == targetLabelValueWildcard { - labelWildcardFound = true + if strings.Contains(k, targetLabelNameWildcard) { + if targetNode.Labels[k] == targetLabelValueWildcard { + labelWildcardFound = true + } + } + if strings.Contains(k, targetLabelNameNegative) { + labelNegativeFound = true } } - if strings.Contains(k, targetLabelNameNegative) { - labelNegativeFound = true - } - } - Expect(labelFound).To(BeTrue(), "label not found!") - Expect(labelWildcardFound).To(BeTrue(), "label for wildcard nodename not found!") - Expect(labelNegativeFound).To(BeFalse(), "label for not existing nodename found!") + Expect(labelFound).To(BeTrue(), "label not found!") + Expect(labelWildcardFound).To(BeTrue(), "label for wildcard nodename not found!") + Expect(labelNegativeFound).To(BeFalse(), "label for not existing nodename found!") - By("Deleting nfd-worker daemonset") - err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Delete(context.TODO(), workerDS.ObjectMeta.Name, metav1.DeleteOptions{}) - Expect(err).NotTo(HaveOccurred()) + By("Deleting nfd-worker daemonset") + err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Delete(context.TODO(), workerDS.ObjectMeta.Name, metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + }) }) - }) - // - // Test NodeFeatureRule - // - Context("and nfd-worker and NodeFeatureRules objects deployed", func() { - It("custom labels from the NodeFeatureRule rules should be created", func() { - By("Creating nfd-worker config") - cm := testutils.NewConfigMap("nfd-worker-conf", "nfd-worker.conf", ` + // + // Test NodeFeatureRule + // + Context("and nfd-worker and NodeFeatureRules objects deployed", func() { + It("custom labels from the NodeFeatureRule rules should be created", func() { + By("Creating nfd-worker config") + cm := testutils.NewConfigMap("nfd-worker-conf", "nfd-worker.conf", ` core: sleepInterval: "1s" featureSources: ["fake"] labelSources: [] `) - cm, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(context.TODO(), cm, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) - - By("Creating nfd-worker daemonset") - podSpecOpts := []testpod.SpecOption{ - testpod.SpecWithContainerImage(dockerImage), - testpod.SpecWithConfigMap(cm.Name, "/etc/kubernetes/node-feature-discovery"), - testpod.SpecWithTolerations(testTolerations), - } - workerDS := testds.NFDWorker(podSpecOpts...) - workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) - - By("Waiting for daemonset pods to be ready") - Expect(testpod.WaitForReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred()) - - expected := map[string]string{ - "feature.node.kubernetes.io/e2e-flag-test-1": "true", - "feature.node.kubernetes.io/e2e-attribute-test-1": "true", - "feature.node.kubernetes.io/e2e-instance-test-1": "true"} - - By("Creating NodeFeatureRules #1") - Expect(testutils.CreateNodeFeatureRulesFromFile(nfdClient, "nodefeaturerule-1.yaml")).NotTo(HaveOccurred()) - - By("Verifying node labels from NodeFeatureRules #1") - Expect(waitForNfdNodeLabels(f.ClientSet, expected)).NotTo(HaveOccurred()) - - By("Creating NodeFeatureRules #2") - Expect(testutils.CreateNodeFeatureRulesFromFile(nfdClient, "nodefeaturerule-2.yaml")).NotTo(HaveOccurred()) - - // Add features from NodeFeatureRule #2 - expected["feature.node.kubernetes.io/e2e-matchany-test-1"] = "true" - expected["feature.node.kubernetes.io/e2e-template-test-1-instance_1"] = "found" - expected["feature.node.kubernetes.io/e2e-template-test-1-instance_2"] = "found" - - By("Verifying node labels from NodeFeatureRules #1 and #2") - Expect(waitForNfdNodeLabels(f.ClientSet, expected)).NotTo(HaveOccurred()) - - // Add features from NodeFeatureRule #3 - By("Creating NodeFeatureRules #3") - Expect(testutils.CreateNodeFeatureRulesFromFile(nfdClient, "nodefeaturerule-3.yaml")).NotTo(HaveOccurred()) - - By("Verifying node taints and annotation from NodeFeatureRules #3") - expectedTaints := []corev1.Taint{ - { - Key: "nfd.node.kubernetes.io/fake-special-node", - Value: "exists", - Effect: "PreferNoSchedule", - }, - { - Key: "nfd.node.kubernetes.io/fake-dedicated-node", - Value: "true", - Effect: "NoExecute", - }, - { - Key: "nfd.node.kubernetes.io/performance-optimized-node", - Value: "true", - Effect: "NoExecute", - }, - } - expectedAnnotation := map[string]string{ - "nfd.node.kubernetes.io/taints": "nfd.node.kubernetes.io/fake-special-node=exists:PreferNoSchedule,nfd.node.kubernetes.io/fake-dedicated-node=true:NoExecute,nfd.node.kubernetes.io/performance-optimized-node=true:NoExecute"} - Expect(waitForNfdNodeTaints(f.ClientSet, expectedTaints)).NotTo(HaveOccurred()) - Expect(waitForNfdNodeAnnotations(f.ClientSet, expectedAnnotation)).NotTo(HaveOccurred()) - - By("Re-applying NodeFeatureRules #3 with updated taints") - Expect(testutils.UpdateNodeFeatureRulesFromFile(nfdClient, "nodefeaturerule-3-updated.yaml")).NotTo(HaveOccurred()) - expectedTaintsUpdated := []corev1.Taint{ - { - Key: "nfd.node.kubernetes.io/fake-special-node", - Value: "exists", - Effect: "PreferNoSchedule", - }, - { - Key: "nfd.node.kubernetes.io/foo", - Value: "true", - Effect: "NoExecute", - }, - } - expectedAnnotationUpdated := map[string]string{ - "nfd.node.kubernetes.io/taints": "nfd.node.kubernetes.io/fake-special-node=exists:PreferNoSchedule,nfd.node.kubernetes.io/foo=true:NoExecute"} + cm, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(context.TODO(), cm, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Creating nfd-worker daemonset") + podSpecOpts := []testpod.SpecOption{ + testpod.SpecWithContainerImage(dockerImage), + testpod.SpecWithConfigMap(cm.Name, "/etc/kubernetes/node-feature-discovery"), + testpod.SpecWithTolerations(testTolerations), + } + workerDS := testds.NFDWorker(podSpecOpts...) + workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for daemonset pods to be ready") + Expect(testpod.WaitForReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred()) + + expected := map[string]string{ + "feature.node.kubernetes.io/e2e-flag-test-1": "true", + "feature.node.kubernetes.io/e2e-attribute-test-1": "true", + "feature.node.kubernetes.io/e2e-instance-test-1": "true"} + + By("Creating NodeFeatureRules #1") + Expect(testutils.CreateNodeFeatureRulesFromFile(nfdClient, "nodefeaturerule-1.yaml")).NotTo(HaveOccurred()) + + By("Verifying node labels from NodeFeatureRules #1") + Expect(waitForNfdNodeLabels(f.ClientSet, expected)).NotTo(HaveOccurred()) + + By("Creating NodeFeatureRules #2") + Expect(testutils.CreateNodeFeatureRulesFromFile(nfdClient, "nodefeaturerule-2.yaml")).NotTo(HaveOccurred()) + + // Add features from NodeFeatureRule #2 + expected["feature.node.kubernetes.io/e2e-matchany-test-1"] = "true" + expected["feature.node.kubernetes.io/e2e-template-test-1-instance_1"] = "found" + expected["feature.node.kubernetes.io/e2e-template-test-1-instance_2"] = "found" + + By("Verifying node labels from NodeFeatureRules #1 and #2") + Expect(waitForNfdNodeLabels(f.ClientSet, expected)).NotTo(HaveOccurred()) + + // Add features from NodeFeatureRule #3 + By("Creating NodeFeatureRules #3") + Expect(testutils.CreateNodeFeatureRulesFromFile(nfdClient, "nodefeaturerule-3.yaml")).NotTo(HaveOccurred()) + + By("Verifying node taints and annotation from NodeFeatureRules #3") + expectedTaints := []corev1.Taint{ + { + Key: "nfd.node.kubernetes.io/fake-special-node", + Value: "exists", + Effect: "PreferNoSchedule", + }, + { + Key: "nfd.node.kubernetes.io/fake-dedicated-node", + Value: "true", + Effect: "NoExecute", + }, + { + Key: "nfd.node.kubernetes.io/performance-optimized-node", + Value: "true", + Effect: "NoExecute", + }, + } + expectedAnnotation := map[string]string{ + "nfd.node.kubernetes.io/taints": "nfd.node.kubernetes.io/fake-special-node=exists:PreferNoSchedule,nfd.node.kubernetes.io/fake-dedicated-node=true:NoExecute,nfd.node.kubernetes.io/performance-optimized-node=true:NoExecute"} + Expect(waitForNfdNodeTaints(f.ClientSet, expectedTaints)).NotTo(HaveOccurred()) + Expect(waitForNfdNodeAnnotations(f.ClientSet, expectedAnnotation)).NotTo(HaveOccurred()) + + By("Re-applying NodeFeatureRules #3 with updated taints") + Expect(testutils.UpdateNodeFeatureRulesFromFile(nfdClient, "nodefeaturerule-3-updated.yaml")).NotTo(HaveOccurred()) + expectedTaintsUpdated := []corev1.Taint{ + { + Key: "nfd.node.kubernetes.io/fake-special-node", + Value: "exists", + Effect: "PreferNoSchedule", + }, + { + Key: "nfd.node.kubernetes.io/foo", + Value: "true", + Effect: "NoExecute", + }, + } + expectedAnnotationUpdated := map[string]string{ + "nfd.node.kubernetes.io/taints": "nfd.node.kubernetes.io/fake-special-node=exists:PreferNoSchedule,nfd.node.kubernetes.io/foo=true:NoExecute"} - By("Verifying updated node taints and annotation from NodeFeatureRules #3") - Expect(waitForNfdNodeTaints(f.ClientSet, expectedTaintsUpdated)).NotTo(HaveOccurred()) - Expect(waitForNfdNodeAnnotations(f.ClientSet, expectedAnnotationUpdated)).NotTo(HaveOccurred()) + By("Verifying updated node taints and annotation from NodeFeatureRules #3") + Expect(waitForNfdNodeTaints(f.ClientSet, expectedTaintsUpdated)).NotTo(HaveOccurred()) + Expect(waitForNfdNodeAnnotations(f.ClientSet, expectedAnnotationUpdated)).NotTo(HaveOccurred()) + }) }) }) + } + + // Run the actual tests + Context("when running NFD with gRPC API enabled", func() { + nfdTestSuite(false) + }) }) From b67d6d7282d9793cefb180c0d5f3d1d6ace24dea Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Wed, 14 Dec 2022 11:46:29 +0200 Subject: [PATCH 65/87] test/e2e: add basic e2e-tests for NodeFeature API Add an initial test set for the NodeFeature API. This is done simply by running a second pass of the tests but with -enable-nodefeature-api (i.e. NodeFeature API enabled and gRPC disabled). This should give basic confidence that the API actually works and form a basis for further imporovements on testing the new CRD API. --- test/e2e/node_feature_discovery.go | 52 ++++++++++++----- test/e2e/utils/pod/pod.go | 2 +- test/e2e/utils/rbac.go | 94 ++++++++++++++++++++++++------ 3 files changed, 116 insertions(+), 32 deletions(-) diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index bb6075bf4f..56157d4474 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -154,14 +154,31 @@ var _ = SIGDescribe("Node Feature Discovery", func() { f := framework.NewDefaultFramework("node-feature-discovery") nfdTestSuite := func(useNodeFeatureApi bool) { + createPodSpecOpts := func(opts ...testpod.SpecOption) []testpod.SpecOption { + if useNodeFeatureApi { + return append(opts, testpod.SpecWithContainerExtraArgs("-enable-nodefeature-api")) + } + return opts + } + Context("when deploying a single nfd-master pod", Ordered, func() { var ( - masterPod *corev1.Pod crds []*apiextensionsv1.CustomResourceDefinition extClient *extclient.Clientset nfdClient *nfdclient.Clientset ) + checkNodeFeatureObject := func(name string) { + _, err := nfdClient.NfdV1alpha1().NodeFeatures(f.Namespace.Name).Get(context.TODO(), name, metav1.GetOptions{}) + if useNodeFeatureApi { + By(fmt.Sprintf("Check that NodeFeature object for the node %q was created", name)) + Expect(err).NotTo(HaveOccurred()) + } else { + By(fmt.Sprintf("Check that NodeFeature object for the node %q hasn't been created", name)) + Expect(err).To(HaveOccurred()) + } + } + BeforeAll(func() { // Create clients for apiextensions and our CRD api extClient = extclient.NewForConfigOrDie(f.ClientConfig()) @@ -198,13 +215,12 @@ var _ = SIGDescribe("Node Feature Discovery", func() { // Launch nfd-master By("Creating nfd master pod and nfd-master service") - - imageOpt := []testpod.SpecOption{ + podSpecOpts := createPodSpecOpts( testpod.SpecWithContainerImage(dockerImage), testpod.SpecWithTolerations(testTolerations), testpod.SpecWithContainerExtraArgs("-enable-taints"), - } - masterPod = e2epod.NewPodClient(f).CreateSync(testpod.NFDMaster(imageOpt...)) + ) + masterPod := e2epod.NewPodClient(f).CreateSync(testpod.NFDMaster(podSpecOpts...)) // Create nfd-master service nfdSvc, err := testutils.CreateService(f.ClientSet, f.Namespace.Name) @@ -247,12 +263,12 @@ var _ = SIGDescribe("Node Feature Discovery", func() { // Launch nfd-worker By("Creating a nfd worker pod") - podSpecOpts := []testpod.SpecOption{ + podSpecOpts := createPodSpecOpts( testpod.SpecWithRestartPolicy(corev1.RestartPolicyNever), testpod.SpecWithContainerImage(dockerImage), testpod.SpecWithContainerExtraArgs("-oneshot", "-label-sources=fake"), testpod.SpecWithTolerations(testTolerations), - } + ) workerPod := testpod.NFDWorker(podSpecOpts...) workerPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), workerPod, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -276,6 +292,8 @@ var _ = SIGDescribe("Node Feature Discovery", func() { } } + checkNodeFeatureObject(node.Name) + By("Deleting the node-feature-discovery worker pod") err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(context.TODO(), workerPod.ObjectMeta.Name, metav1.DeleteOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -299,10 +317,10 @@ var _ = SIGDescribe("Node Feature Discovery", func() { fConf := cfg.DefaultFeatures By("Creating nfd-worker daemonset") - podSpecOpts := []testpod.SpecOption{ + podSpecOpts := createPodSpecOpts( testpod.SpecWithContainerImage(dockerImage), testpod.SpecWithTolerations(testTolerations), - } + ) workerDS := testds.NFDWorker(podSpecOpts...) workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -364,6 +382,9 @@ var _ = SIGDescribe("Node Feature Discovery", func() { } } + // Check existence of NodeFeature object + checkNodeFeatureObject(node.Name) + } By("Deleting nfd-worker daemonset") @@ -426,12 +447,12 @@ var _ = SIGDescribe("Node Feature Discovery", func() { Expect(err).NotTo(HaveOccurred()) By("Creating nfd-worker daemonset with configmap mounted") - podSpecOpts := []testpod.SpecOption{ + podSpecOpts := createPodSpecOpts( testpod.SpecWithContainerImage(dockerImage), testpod.SpecWithConfigMap(cm1.Name, filepath.Join(custom.Directory, "cm1")), testpod.SpecWithConfigMap(cm2.Name, filepath.Join(custom.Directory, "cm2")), testpod.SpecWithTolerations(testTolerations), - } + ) workerDS := testds.NFDWorker(podSpecOpts...) workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) @@ -489,11 +510,11 @@ core: Expect(err).NotTo(HaveOccurred()) By("Creating nfd-worker daemonset") - podSpecOpts := []testpod.SpecOption{ + podSpecOpts := createPodSpecOpts( testpod.SpecWithContainerImage(dockerImage), testpod.SpecWithConfigMap(cm.Name, "/etc/kubernetes/node-feature-discovery"), testpod.SpecWithTolerations(testTolerations), - } + ) workerDS := testds.NFDWorker(podSpecOpts...) workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -580,6 +601,11 @@ core: nfdTestSuite(false) }) + + Context("when running NFD with NodeFeature CRD API enabled", func() { + nfdTestSuite(true) + }) + }) // waitForNfdNodeAnnotations waits for node to be annotated as expected. diff --git a/test/e2e/utils/pod/pod.go b/test/e2e/utils/pod/pod.go index a5f0381af9..5a93b19f68 100644 --- a/test/e2e/utils/pod/pod.go +++ b/test/e2e/utils/pod/pod.go @@ -307,7 +307,7 @@ func nfdWorkerSpec(opts ...SpecOption) *corev1.PodSpec { }, }, }, - ServiceAccountName: "nfd-master-e2e", + ServiceAccountName: "nfd-worker-e2e", DNSPolicy: corev1.DNSClusterFirstWithHostNet, Volumes: []corev1.Volume{ { diff --git a/test/e2e/utils/rbac.go b/test/e2e/utils/rbac.go index 9d851576b1..e71db751e2 100644 --- a/test/e2e/utils/rbac.go +++ b/test/e2e/utils/rbac.go @@ -32,12 +32,17 @@ var ( // ConfigureRBAC creates required RBAC configuration func ConfigureRBAC(cs clientset.Interface, ns string) error { - _, err := createServiceAccountMaster(cs, ns) + _, err := createServiceAccount(cs, "nfd-master-e2e", ns) if err != nil { return err } - _, err = createServiceAccountTopologyUpdater(cs, ns) + _, err = createServiceAccount(cs, "nfd-worker-e2e", ns) + if err != nil { + return err + } + + _, err = createServiceAccount(cs, "nfd-topology-updater-e2e", ns) if err != nil { return err } @@ -47,6 +52,11 @@ func ConfigureRBAC(cs clientset.Interface, ns string) error { return err } + _, err = createRoleWorker(cs, ns) + if err != nil { + return err + } + _, err = createClusterRoleTopologyUpdater(cs) if err != nil { return err @@ -57,6 +67,11 @@ func ConfigureRBAC(cs clientset.Interface, ns string) error { return err } + _, err = createRoleBindingWorker(cs, ns) + if err != nil { + return err + } + _, err = createClusterRoleBindingTopologyUpdater(cs, ns) if err != nil { return err @@ -75,6 +90,10 @@ func DeconfigureRBAC(cs clientset.Interface, ns string) error { if err != nil { return err } + err = cs.RbacV1().RoleBindings(ns).Delete(context.TODO(), "nfd-worker-e2e", metav1.DeleteOptions{}) + if err != nil { + return err + } err = cs.RbacV1().ClusterRoles().Delete(context.TODO(), "nfd-topology-updater-e2e", metav1.DeleteOptions{}) if err != nil { return err @@ -83,6 +102,10 @@ func DeconfigureRBAC(cs clientset.Interface, ns string) error { if err != nil { return err } + err = cs.RbacV1().Roles(ns).Delete(context.TODO(), "nfd-worker-e2e", metav1.DeleteOptions{}) + if err != nil { + return err + } err = cs.CoreV1().ServiceAccounts(ns).Delete(context.TODO(), "nfd-topology-updater-e2e", metav1.DeleteOptions{}) if err != nil { return err @@ -91,25 +114,18 @@ func DeconfigureRBAC(cs clientset.Interface, ns string) error { if err != nil { return err } - return nil -} - -// Configure service account required by NFD Master -func createServiceAccountMaster(cs clientset.Interface, ns string) (*corev1.ServiceAccount, error) { - sa := &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Name: "nfd-master-e2e", - Namespace: ns, - }, + err = cs.CoreV1().ServiceAccounts(ns).Delete(context.TODO(), "nfd-worker-e2e", metav1.DeleteOptions{}) + if err != nil { + return err } - return cs.CoreV1().ServiceAccounts(ns).Create(context.TODO(), sa, metav1.CreateOptions{}) + return nil } -// Configure service account required by NFD MTopology Updater -func createServiceAccountTopologyUpdater(cs clientset.Interface, ns string) (*corev1.ServiceAccount, error) { +// Configure service account +func createServiceAccount(cs clientset.Interface, name, ns string) (*corev1.ServiceAccount, error) { sa := &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ - Name: "nfd-topology-updater-e2e", + Name: name, Namespace: ns, }, } @@ -126,11 +142,11 @@ func createClusterRoleMaster(cs clientset.Interface) (*rbacv1.ClusterRole, error { APIGroups: []string{""}, Resources: []string{"nodes"}, - Verbs: []string{"get", "patch", "update"}, + Verbs: []string{"get", "list", "patch", "update"}, }, { APIGroups: []string{"nfd.k8s-sigs.io"}, - Resources: []string{"nodefeaturerules"}, + Resources: []string{"nodefeatures", "nodefeaturerules"}, Verbs: []string{"get", "list", "watch"}, }, }, @@ -148,6 +164,24 @@ func createClusterRoleMaster(cs clientset.Interface) (*rbacv1.ClusterRole, error return cs.RbacV1().ClusterRoles().Update(context.TODO(), cr, metav1.UpdateOptions{}) } +// Configure role required by NFD Worker +func createRoleWorker(cs clientset.Interface, ns string) (*rbacv1.Role, error) { + cr := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nfd-worker-e2e", + Namespace: ns, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"nfd.k8s-sigs.io"}, + Resources: []string{"nodefeatures"}, + Verbs: []string{"create", "get", "update"}, + }, + }, + } + return cs.RbacV1().Roles(ns).Update(context.TODO(), cr, metav1.UpdateOptions{}) +} + // Configure cluster role required by NFD Topology Updater func createClusterRoleTopologyUpdater(cs clientset.Interface) (*rbacv1.ClusterRole, error) { cr := &rbacv1.ClusterRole{ @@ -210,6 +244,30 @@ func createClusterRoleBindingMaster(cs clientset.Interface, ns string) (*rbacv1. return cs.RbacV1().ClusterRoleBindings().Update(context.TODO(), crb, metav1.UpdateOptions{}) } +// Configure role binding required by NFD Master +func createRoleBindingWorker(cs clientset.Interface, ns string) (*rbacv1.RoleBinding, error) { + crb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nfd-worker-e2e", + Namespace: ns, + }, + Subjects: []rbacv1.Subject{ + { + Kind: rbacv1.ServiceAccountKind, + Name: "nfd-worker-e2e", + Namespace: ns, + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "Role", + Name: "nfd-worker-e2e", + }, + } + + return cs.RbacV1().RoleBindings(ns).Update(context.TODO(), crb, metav1.UpdateOptions{}) +} + // Configure cluster role binding required by NFD Topology Updater func createClusterRoleBindingTopologyUpdater(cs clientset.Interface, ns string) (*rbacv1.ClusterRoleBinding, error) { crb := &rbacv1.ClusterRoleBinding{ From f5ae3fe2c7b2140091cc3eeb022d12e36bdb412c Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Mon, 19 Dec 2022 17:00:35 +0200 Subject: [PATCH 66/87] Simplify usage of ObjectMeta fields No need to explicitly spell out ObjectMeta as it's embedded in the object types. --- pkg/nfd-master/nfd-master.go | 4 ++-- test/e2e/node_feature_discovery.go | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/nfd-master/nfd-master.go b/pkg/nfd-master/nfd-master.go index 4ca73c6168..7446471c2e 100644 --- a/pkg/nfd-master/nfd-master.go +++ b/pkg/nfd-master/nfd-master.go @@ -722,10 +722,10 @@ func (m *nfdMaster) processNodeFeatureRule(features *nfdv1alpha1.Features) (map[ for _, spec := range ruleSpecs { switch { case klog.V(3).Enabled(): - h := fmt.Sprintf("executing NodeFeatureRule %q:", spec.ObjectMeta.Name) + h := fmt.Sprintf("executing NodeFeatureRule %q:", spec.Name) utils.KlogDump(3, h, " ", spec.Spec) case klog.V(1).Enabled(): - klog.Infof("executing NodeFeatureRule %q", spec.ObjectMeta.Name) + klog.Infof("executing NodeFeatureRule %q", spec.Name) } for _, rule := range spec.Spec.Rules { ruleOut, err := rule.Execute(features) diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index 56157d4474..5ba26404a8 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -231,7 +231,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { By("Verifying the node where nfd-master is running") // Get updated masterPod object (we want to know where it was scheduled) - masterPod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(context.TODO(), masterPod.ObjectMeta.Name, metav1.GetOptions{}) + masterPod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(context.TODO(), masterPod.Name, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) // Node running nfd-master should have master version annotation masterPodNode, err := f.ClientSet.CoreV1().Nodes().Get(context.TODO(), masterPod.Spec.NodeName, metav1.GetOptions{}) @@ -239,7 +239,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { Expect(masterPodNode.Annotations).To(HaveKey(nfdv1alpha1.AnnotationNs + "/master.version")) By("Waiting for the nfd-master service to be up") - Expect(e2enetwork.WaitForService(f.ClientSet, f.Namespace.Name, nfdSvc.ObjectMeta.Name, true, time.Second, 10*time.Second)).NotTo(HaveOccurred()) + Expect(e2enetwork.WaitForService(f.ClientSet, f.Namespace.Name, nfdSvc.Name, true, time.Second, 10*time.Second)).NotTo(HaveOccurred()) }) AfterEach(func() { @@ -274,8 +274,8 @@ var _ = SIGDescribe("Node Feature Discovery", func() { Expect(err).NotTo(HaveOccurred()) By("Waiting for the nfd-worker pod to succeed") - Expect(e2epod.WaitForPodSuccessInNamespace(f.ClientSet, workerPod.ObjectMeta.Name, f.Namespace.Name)).NotTo(HaveOccurred()) - workerPod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(context.TODO(), workerPod.ObjectMeta.Name, metav1.GetOptions{}) + Expect(e2epod.WaitForPodSuccessInNamespace(f.ClientSet, workerPod.Name, f.Namespace.Name)).NotTo(HaveOccurred()) + workerPod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(context.TODO(), workerPod.Name, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) By(fmt.Sprintf("Making sure '%s' was decorated with the fake feature labels", workerPod.Spec.NodeName)) @@ -295,7 +295,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { checkNodeFeatureObject(node.Name) By("Deleting the node-feature-discovery worker pod") - err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(context.TODO(), workerPod.ObjectMeta.Name, metav1.DeleteOptions{}) + err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(context.TODO(), workerPod.Name, metav1.DeleteOptions{}) Expect(err).NotTo(HaveOccurred()) }) }) @@ -388,7 +388,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { } By("Deleting nfd-worker daemonset") - err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Delete(context.TODO(), workerDS.ObjectMeta.Name, metav1.DeleteOptions{}) + err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Delete(context.TODO(), workerDS.Name, metav1.DeleteOptions{}) Expect(err).NotTo(HaveOccurred()) }) }) @@ -489,7 +489,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { Expect(labelNegativeFound).To(BeFalse(), "label for not existing nodename found!") By("Deleting nfd-worker daemonset") - err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Delete(context.TODO(), workerDS.ObjectMeta.Name, metav1.DeleteOptions{}) + err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Delete(context.TODO(), workerDS.Name, metav1.DeleteOptions{}) Expect(err).NotTo(HaveOccurred()) }) }) From 27c47bd088ee529ab16ea620322d1fc0b71deba2 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Tue, 20 Dec 2022 15:14:41 +0200 Subject: [PATCH 67/87] docs: better document differences between deployment methods --- docs/deployment/helm.md | 4 ++++ docs/deployment/index.md | 12 +++++++++++- docs/deployment/kustomize.md | 10 ++++++++-- docs/deployment/operator.md | 5 +++++ 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/deployment/helm.md b/docs/deployment/helm.md index 49ab7052c6..f618d76151 100644 --- a/docs/deployment/helm.md +++ b/docs/deployment/helm.md @@ -17,6 +17,10 @@ sort: 3 Node Feature Discovery Helm chart allow to easily deploy and manage NFD. +> NOTE: NFD is not ideal for other Helm charts to depend on as that may result +> in multiple parallel NFD deployments in the same cluster which is not fully +> supported by the NFD Helm chart. + ## Prerequisites [Helm package manager](https://helm.sh/) should be installed. diff --git a/docs/deployment/index.md b/docs/deployment/index.md index 2b5f5c9fe2..9f4fb64cc5 100644 --- a/docs/deployment/index.md +++ b/docs/deployment/index.md @@ -6,4 +6,14 @@ sort: 2 # Deployment -Deployment instructions. +See [Image variants](image-variants) for description of the different NFD +container images available. + +[Using Kustomize](kustomize) provides straightforward deployment with +`kubectl` integration and declarative customization. + +[Using Helm](helm) provides easy management of NFD deployments with nice +configuration management and easy upgrades. + +[Using Operator](operator) provides deployment and configuration management via +CRDs. diff --git a/docs/deployment/kustomize.md b/docs/deployment/kustomize.md index b5d84a0d74..efb6678beb 100644 --- a/docs/deployment/kustomize.md +++ b/docs/deployment/kustomize.md @@ -15,9 +15,15 @@ sort: 2 --- -This deployment method requires +[Kustomize](https://github.com/kubernetes-sigs/kustomize) provides easy +deployment of NFD. Customization of the deployment is done by maintaining +declarative overlays on top of the base overlays in NFD. + +To follow the deployment instructions here, [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl) v1.21 or -later. The kustomize overlays provided in the repo can be used directly: +later is required. + +The kustomize overlays provided in the repo can be used directly: ```bash kubectl apply -k https://github.com/kubernetes-sigs/node-feature-discovery/deployment/overlays/default?ref={{ site.release }} diff --git a/docs/deployment/operator.md b/docs/deployment/operator.md index b9d47ff09b..df1c7846b9 100644 --- a/docs/deployment/operator.md +++ b/docs/deployment/operator.md @@ -15,6 +15,11 @@ sort: 4 --- +The [Node Feature Discovery Operator][nfd-operator] automates installation, +configuration and updates of NFD using a specific NodeFeatureDiscovery custom +resource. This also provides good support for managing NFD as a dependency of +other operators. + ## Deployment Deployment using the From 30e7c247950b171aa425d0fb801dfdc9c33c7ad6 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Wed, 21 Dec 2022 10:24:48 +0200 Subject: [PATCH 68/87] github: update new-release issue template Add instructions about creating prow jobs for new release branches. --- .github/ISSUE_TEMPLATE/new-release.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/new-release.md b/.github/ISSUE_TEMPLATE/new-release.md index ef10d67cd4..80f4c3dbbe 100644 --- a/.github/ISSUE_TEMPLATE/new-release.md +++ b/.github/ISSUE_TEMPLATE/new-release.md @@ -17,6 +17,9 @@ Please do not remove items from the checklist `git branch release-0.$MAJ master` - [ ] An OWNER pushes the new release branch with `git push release-0.$MAJ` + - [ ] Create Prow pre-submit job configuration for the new release branch in K8s + [test-infra](https://github.com/kubernetes/test-infra), submit a PR + - [ ] Wait for the test-infra Prow config PR to be merged - [ ] Run `hack/prepare-release.sh $VERSION` to turn references to point to the upcoming release (README, deployment templates, docs configuration, test/e2e flags), submit a PR against the release branch - An OWNER prepares a draft release From b91922746a9c4a2b21f05f894e46b3341e6a3fca Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Wed, 21 Dec 2022 13:26:31 +0200 Subject: [PATCH 69/87] docs: mention NodeFeature as an extension point In the CRD intro, mention that NodeFeature can be used as an integration point for 3rd party extensions. --- docs/usage/custom-resources.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/usage/custom-resources.md b/docs/usage/custom-resources.md index deec64dec1..8a4853d406 100644 --- a/docs/usage/custom-resources.md +++ b/docs/usage/custom-resources.md @@ -24,7 +24,10 @@ NodeFeatureRule is an NFD-specific custom resource for communicating node features and node labeling requests. Support for NodeFeatureRule objects is disabled by default. If enabled, nfd-master watches for NodeFeature objects, labels nodes accordingly and uses the listed features as input when evaluating -[NodeFeatureRule](#nodefeaturerule)s. +[NodeFeatureRule](#nodefeaturerule)s. NodeFeature objects can be used for +implementing 3rd party extensions (see +[customization guide](customization-guide#nodefeature-custom-resource) for more +details). ```yaml apiVersion: nfd.k8s-sigs.io/v1alpha1 From 8c0e38d0c5985182af68c6031e7f804996421494 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Wed, 21 Dec 2022 13:42:10 +0200 Subject: [PATCH 70/87] docs: fix typo in CRD name --- docs/usage/custom-resources.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/usage/custom-resources.md b/docs/usage/custom-resources.md index 8a4853d406..1fd64d0ea0 100644 --- a/docs/usage/custom-resources.md +++ b/docs/usage/custom-resources.md @@ -20,10 +20,10 @@ NFD uses some Kubernetes [custom resources][custom-resources]. ## NodeFeature **EXPERIMENTAL** -NodeFeatureRule is an NFD-specific custom resource for communicating node -features and node labeling requests. Support for NodeFeatureRule objects is +NodeFeature is an NFD-specific custom resource for communicating node +features and node labeling requests. Support for NodeFeature objects is disabled by default. If enabled, nfd-master watches for NodeFeature objects, -labels nodes accordingly and uses the listed features as input when evaluating +labels nodes as specified and uses the listed features as input when evaluating [NodeFeatureRule](#nodefeaturerule)s. NodeFeature objects can be used for implementing 3rd party extensions (see [customization guide](customization-guide#nodefeature-custom-resource) for more From 59a2757115e6a07ecca52514123092d0bc389ee5 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Wed, 21 Dec 2022 14:52:56 +0200 Subject: [PATCH 71/87] Use single-dash format for nfd cmdline flags Use the "single-dash" version of nfd command line flags in deployment files and e2e-tests. No impact in functionality, just aligns with documentation and other parts of the codebase. --- .../node-feature-discovery/templates/master.yaml | 12 ++++++------ .../templates/topologyupdater.yaml | 14 +++++++------- .../node-feature-discovery/templates/worker.yaml | 8 ++++---- deployment/overlays/prune/master-job.yaml | 2 +- test/e2e/utils/pod/pod.go | 8 ++++---- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/deployment/helm/node-feature-discovery/templates/master.yaml b/deployment/helm/node-feature-discovery/templates/master.yaml index 26a326cb80..220b8af9b2 100644 --- a/deployment/helm/node-feature-discovery/templates/master.yaml +++ b/deployment/helm/node-feature-discovery/templates/master.yaml @@ -76,16 +76,16 @@ spec: {{- toYaml .Values.master.resources | nindent 12 }} args: {{- if .Values.master.instance | empty | not }} - - "--instance={{ .Values.master.instance }}" + - "-instance={{ .Values.master.instance }}" {{- end }} {{- if .Values.enableNodeFeatureApi }} - "-enable-nodefeature-api" {{- end }} {{- if .Values.master.extraLabelNs | empty | not }} - - "--extra-label-ns={{- join "," .Values.master.extraLabelNs }}" + - "-extra-label-ns={{- join "," .Values.master.extraLabelNs }}" {{- end }} {{- if .Values.master.resourceLabels | empty | not }} - - "--resource-labels={{- join "," .Values.master.resourceLabels }}" + - "-resource-labels={{- join "," .Values.master.resourceLabels }}" {{- end }} {{- if .Values.master.crdController | kindIs "invalid" | not }} - "-crd-controller={{ .Values.master.crdController }}" @@ -97,9 +97,9 @@ spec: - "-featurerules-controller={{ .Values.master.featureRulesController }}" {{- end }} {{- if .Values.tls.enable }} - - "--ca-file=/etc/kubernetes/node-feature-discovery/certs/ca.crt" - - "--key-file=/etc/kubernetes/node-feature-discovery/certs/tls.key" - - "--cert-file=/etc/kubernetes/node-feature-discovery/certs/tls.crt" + - "-ca-file=/etc/kubernetes/node-feature-discovery/certs/ca.crt" + - "-key-file=/etc/kubernetes/node-feature-discovery/certs/tls.key" + - "-cert-file=/etc/kubernetes/node-feature-discovery/certs/tls.crt" volumeMounts: - name: nfd-master-cert mountPath: "/etc/kubernetes/node-feature-discovery/certs" diff --git a/deployment/helm/node-feature-discovery/templates/topologyupdater.yaml b/deployment/helm/node-feature-discovery/templates/topologyupdater.yaml index 2f28b969f2..8f4849ac5a 100644 --- a/deployment/helm/node-feature-discovery/templates/topologyupdater.yaml +++ b/deployment/helm/node-feature-discovery/templates/topologyupdater.yaml @@ -41,19 +41,19 @@ spec: - "nfd-topology-updater" args: {{- if .Values.topologyUpdater.updateInterval | empty | not }} - - "--sleep-interval={{ .Values.topologyUpdater.updateInterval }}" + - "-sleep-interval={{ .Values.topologyUpdater.updateInterval }}" {{- else }} - - "--sleep-interval=3s" + - "-sleep-interval=3s" {{- end }} {{- if .Values.topologyUpdater.watchNamespace | empty | not }} - - "--watch-namespace={{ .Values.topologyUpdater.watchNamespace }}" + - "-watch-namespace={{ .Values.topologyUpdater.watchNamespace }}" {{- else }} - - "--watch-namespace=*" + - "-watch-namespace=*" {{- end }} {{- if .Values.tls.enable }} - - "--ca-file=/etc/kubernetes/node-feature-discovery/certs/ca.crt" - - "--key-file=/etc/kubernetes/node-feature-discovery/certs/tls.key" - - "--cert-file=/etc/kubernetes/node-feature-discovery/certs/tls.crt" + - "-ca-file=/etc/kubernetes/node-feature-discovery/certs/ca.crt" + - "-key-file=/etc/kubernetes/node-feature-discovery/certs/tls.key" + - "-cert-file=/etc/kubernetes/node-feature-discovery/certs/tls.crt" {{- end }} volumeMounts: - name: kubelet-config diff --git a/deployment/helm/node-feature-discovery/templates/worker.yaml b/deployment/helm/node-feature-discovery/templates/worker.yaml index e723cc5cbe..829e4e5e36 100644 --- a/deployment/helm/node-feature-discovery/templates/worker.yaml +++ b/deployment/helm/node-feature-discovery/templates/worker.yaml @@ -45,14 +45,14 @@ spec: command: - "nfd-worker" args: - - "--server={{ include "node-feature-discovery.fullname" . }}-master:{{ .Values.master.service.port }}" + - "-server={{ include "node-feature-discovery.fullname" . }}-master:{{ .Values.master.service.port }}" {{- if .Values.enableNodeFeatureApi }} - "-enable-nodefeature-api" {{- end }} {{- if .Values.tls.enable }} - - "--ca-file=/etc/kubernetes/node-feature-discovery/certs/ca.crt" - - "--key-file=/etc/kubernetes/node-feature-discovery/certs/tls.key" - - "--cert-file=/etc/kubernetes/node-feature-discovery/certs/tls.crt" + - "-ca-file=/etc/kubernetes/node-feature-discovery/certs/ca.crt" + - "-key-file=/etc/kubernetes/node-feature-discovery/certs/tls.key" + - "-cert-file=/etc/kubernetes/node-feature-discovery/certs/tls.crt" {{- end }} volumeMounts: - name: host-boot diff --git a/deployment/overlays/prune/master-job.yaml b/deployment/overlays/prune/master-job.yaml index 9ea1c24d36..247b59a5f0 100644 --- a/deployment/overlays/prune/master-job.yaml +++ b/deployment/overlays/prune/master-job.yaml @@ -20,6 +20,6 @@ spec: command: - "nfd-master" args: - - "--prune" + - "-prune" restartPolicy: Never diff --git a/test/e2e/utils/pod/pod.go b/test/e2e/utils/pod/pod.go index 5a93b19f68..ec719f331b 100644 --- a/test/e2e/utils/pod/pod.go +++ b/test/e2e/utils/pod/pod.go @@ -363,10 +363,10 @@ func NFDTopologyUpdaterSpec(kc utils.KubeletConfig, opts ...SpecOption) *corev1. ImagePullPolicy: pullPolicy(), Command: []string{"nfd-topology-updater"}, Args: []string{ - "--kubelet-config-uri=file:///podresources/config.yaml", - "--podresources-socket=unix:///podresources/kubelet.sock", - "--sleep-interval=3s", - "--watch-namespace=rte"}, + "-kubelet-config-uri=file:///podresources/config.yaml", + "-podresources-socket=unix:///podresources/kubelet.sock", + "-sleep-interval=3s", + "-watch-namespace=rte"}, Env: []corev1.EnvVar{ { Name: "NODE_NAME", From 1938f9fd87344e8113ce3e4febeb1a3ba4734fb1 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Wed, 21 Dec 2022 16:10:14 +0200 Subject: [PATCH 72/87] README: update to latest release v0.12.0 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bb918f3a8f..44d7f23f8b 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ features and system configuration! #### Quick-start – the short-short version ```bash -$ kubectl apply -k https://github.com/kubernetes-sigs/node-feature-discovery/deployment/overlays/default?ref=v0.11.3 +$ kubectl apply -k https://github.com/kubernetes-sigs/node-feature-discovery/deployment/overlays/default?ref=v0.12.0 namespace/node-feature-discovery created customresourcedefinition.apiextensions.k8s.io/nodefeaturerules.nfd.k8s-sigs.io created serviceaccount/nfd-master created From fe0d706c8b94f48fdaa22a1cfa5e746070601546 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Wed, 21 Dec 2022 20:18:13 +0200 Subject: [PATCH 73/87] dockerfile: update grpc-health-probe to v0.4.14 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f1b9106a3e..40c4af2667 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ ARG BASE_IMAGE_MINIMAL FROM ${BUILDER_IMAGE} as builder # Build and install the grpc-health-probe binary -RUN GRPC_HEALTH_PROBE_VERSION=v0.4.6 && \ +RUN GRPC_HEALTH_PROBE_VERSION=v0.4.14 && \ go install github.com/grpc-ecosystem/grpc-health-probe@${GRPC_HEALTH_PROBE_VERSION} \ # Rename it as it's referenced as grpc_health_probe in the deployment yamls # and in its own project https://github.com/grpc-ecosystem/grpc-health-probe From a4fc15a424f5283f2319921d80d8355433bf73e3 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Thu, 22 Dec 2022 17:53:42 +0200 Subject: [PATCH 74/87] apis/nfd: make all fields in NodeFeatureSpec optional Don't require features to be specified. The creator possibly only wants to create labels or only some types of features. No need to specify empty structs for the unused fields. --- pkg/apis/nfd/v1alpha1/types.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/apis/nfd/v1alpha1/types.go b/pkg/apis/nfd/v1alpha1/types.go index 3565db1cd0..c916daa86d 100644 --- a/pkg/apis/nfd/v1alpha1/types.go +++ b/pkg/apis/nfd/v1alpha1/types.go @@ -46,6 +46,7 @@ type NodeFeature struct { // NodeFeatureSpec describes a NodeFeature object. type NodeFeatureSpec struct { // Features is the full "raw" features data that has been discovered. + // +optional Features Features `json:"features"` // Labels is the set of node labels that are requested to be created. // +optional @@ -56,9 +57,15 @@ type NodeFeatureSpec struct { // // +protobuf=true type Features struct { - Flags map[string]FlagFeatureSet `json:"flags" protobuf:"bytes,1,rep,name=flags"` + // Flags contains all the flag-type features of the node. + // +optional + Flags map[string]FlagFeatureSet `json:"flags" protobuf:"bytes,1,rep,name=flags"` + // Attributes contains all the attribute-type features of the node. + // +optional Attributes map[string]AttributeFeatureSet `json:"attributes" protobuf:"bytes,2,rep,name=vattributes"` - Instances map[string]InstanceFeatureSet `json:"instances" protobuf:"bytes,3,rep,name=instances"` + // Instances contains all the instance-type features of the node. + // +optional + Instances map[string]InstanceFeatureSet `json:"instances" protobuf:"bytes,3,rep,name=instances"` } // FlagFeatureSet is a set of simple features only containing names without values. From dfda9bccadb86a22927dbb61896b8f399bf546bf Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Thu, 22 Dec 2022 17:58:20 +0200 Subject: [PATCH 75/87] apis/nfd: update auto-generated code --- deployment/base/nfd-crds/nfd-api-crds.yaml | 12 ++++++------ .../node-feature-discovery/crds/nfd-api-crds.yaml | 12 ++++++------ pkg/apis/nfd/v1alpha1/generated.proto | 6 ++++++ 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/deployment/base/nfd-crds/nfd-api-crds.yaml b/deployment/base/nfd-crds/nfd-api-crds.yaml index ab10f15a9c..5d9e1e16d4 100644 --- a/deployment/base/nfd-crds/nfd-api-crds.yaml +++ b/deployment/base/nfd-crds/nfd-api-crds.yaml @@ -52,6 +52,8 @@ spec: required: - elements type: object + description: Attributes contains all the attribute-type features + of the node. type: object flags: additionalProperties: @@ -67,6 +69,8 @@ spec: required: - elements type: object + description: Flags contains all the flag-type features of the + node. type: object instances: additionalProperties: @@ -89,11 +93,9 @@ spec: required: - elements type: object + description: Instances contains all the instance-type features + of the node. type: object - required: - - attributes - - flags - - instances type: object labels: additionalProperties: @@ -101,8 +103,6 @@ spec: description: Labels is the set of node labels that are requested to be created. type: object - required: - - features type: object required: - spec diff --git a/deployment/helm/node-feature-discovery/crds/nfd-api-crds.yaml b/deployment/helm/node-feature-discovery/crds/nfd-api-crds.yaml index ab10f15a9c..5d9e1e16d4 100644 --- a/deployment/helm/node-feature-discovery/crds/nfd-api-crds.yaml +++ b/deployment/helm/node-feature-discovery/crds/nfd-api-crds.yaml @@ -52,6 +52,8 @@ spec: required: - elements type: object + description: Attributes contains all the attribute-type features + of the node. type: object flags: additionalProperties: @@ -67,6 +69,8 @@ spec: required: - elements type: object + description: Flags contains all the flag-type features of the + node. type: object instances: additionalProperties: @@ -89,11 +93,9 @@ spec: required: - elements type: object + description: Instances contains all the instance-type features + of the node. type: object - required: - - attributes - - flags - - instances type: object labels: additionalProperties: @@ -101,8 +103,6 @@ spec: description: Labels is the set of node labels that are requested to be created. type: object - required: - - features type: object required: - spec diff --git a/pkg/apis/nfd/v1alpha1/generated.proto b/pkg/apis/nfd/v1alpha1/generated.proto index 09f408ee76..e3d26de3d3 100644 --- a/pkg/apis/nfd/v1alpha1/generated.proto +++ b/pkg/apis/nfd/v1alpha1/generated.proto @@ -35,10 +35,16 @@ message AttributeFeatureSet { // // +protobuf=true message Features { + // Flags contains all the flag-type features of the node. + // +optional map flags = 1; + // Attributes contains all the attribute-type features of the node. + // +optional map vattributes = 2; + // Instances contains all the instance-type features of the node. + // +optional map instances = 3; } From aa97105854f526a2a6fb5034f800fc991b13349a Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Fri, 23 Dec 2022 09:50:15 +0200 Subject: [PATCH 76/87] Add common utility function for getting node name --- cmd/nfd-topology-updater/main.go | 5 ++--- pkg/nfd-client/base.go | 9 --------- .../topology-updater/nfd-topology-updater.go | 2 +- pkg/nfd-client/worker/nfd-worker.go | 6 +++--- pkg/nfd-master/nfd-master.go | 3 +-- pkg/utils/kubernetes.go | 10 ++++++++++ source/system/system.go | 2 +- 7 files changed, 18 insertions(+), 19 deletions(-) diff --git a/cmd/nfd-topology-updater/main.go b/cmd/nfd-topology-updater/main.go index e6115b3ee1..0db443e27f 100644 --- a/cmd/nfd-topology-updater/main.go +++ b/cmd/nfd-topology-updater/main.go @@ -26,7 +26,6 @@ import ( "k8s.io/klog/v2" kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" - nfdclient "sigs.k8s.io/node-feature-discovery/pkg/nfd-client" topology "sigs.k8s.io/node-feature-discovery/pkg/nfd-client/topology-updater" "sigs.k8s.io/node-feature-discovery/pkg/resourcemonitor" "sigs.k8s.io/node-feature-discovery/pkg/topologypolicy" @@ -111,12 +110,12 @@ func parseArgs(flags *flag.FlagSet, osArgs ...string) (*topology.Args, *resource } if len(resourcemonitorArgs.KubeletConfigURI) == 0 { - if len(nfdclient.NodeName()) == 0 { + if len(utils.NodeName()) == 0 { fmt.Fprintf(flags.Output(), "unable to determine the default kubelet config endpoint 'https://${NODE_NAME}:%d/configz' due to empty NODE_NAME environment, "+ "please either define the NODE_NAME environment variable or specify endpoint with the -kubelet-config-uri flag\n", kubeletSecurePort) os.Exit(1) } - resourcemonitorArgs.KubeletConfigURI = fmt.Sprintf("https://%s:%d/configz", nfdclient.NodeName(), kubeletSecurePort) + resourcemonitorArgs.KubeletConfigURI = fmt.Sprintf("https://%s:%d/configz", utils.NodeName(), kubeletSecurePort) } return args, resourcemonitorArgs diff --git a/pkg/nfd-client/base.go b/pkg/nfd-client/base.go index c965535421..e8f53ed059 100644 --- a/pkg/nfd-client/base.go +++ b/pkg/nfd-client/base.go @@ -56,15 +56,6 @@ type Args struct { Klog map[string]*utils.KlogFlagVal } -var nodeName string - -func init() { - nodeName = os.Getenv("NODE_NAME") -} - -// NodeName returns the name of the k8s node we're running on. -func NodeName() string { return nodeName } - // NewNfdBaseClient creates a new NfdBaseClient instance. func NewNfdBaseClient(args *Args) (NfdBaseClient, error) { nfd := NfdBaseClient{args: *args} diff --git a/pkg/nfd-client/topology-updater/nfd-topology-updater.go b/pkg/nfd-client/topology-updater/nfd-topology-updater.go index 7ada9cb3ba..d3072bde23 100644 --- a/pkg/nfd-client/topology-updater/nfd-topology-updater.go +++ b/pkg/nfd-client/topology-updater/nfd-topology-updater.go @@ -78,7 +78,7 @@ func NewTopologyUpdater(args Args, resourcemonitorArgs resourcemonitor.Args, pol args: args, resourcemonitorArgs: resourcemonitorArgs, nodeInfo: &staticNodeInfo{ - nodeName: os.Getenv("NODE_NAME"), + nodeName: utils.NodeName(), tmPolicy: policy, }, stop: make(chan struct{}, 1), diff --git a/pkg/nfd-client/worker/nfd-worker.go b/pkg/nfd-client/worker/nfd-worker.go index abc0fdb4a8..1f7443cf0e 100644 --- a/pkg/nfd-client/worker/nfd-worker.go +++ b/pkg/nfd-client/worker/nfd-worker.go @@ -157,7 +157,7 @@ func newDefaultConfig() *NFDConfig { // one request if OneShot is set to 'true' in the worker args. func (w *nfdWorker) Run() error { klog.Infof("Node Feature Discovery Worker %s", version.Get()) - klog.Infof("NodeName: '%s'", clientcommon.NodeName()) + klog.Infof("NodeName: '%s'", utils.NodeName()) klog.Infof("Kubernetes namespace: '%s'", w.kubernetesNamespace) // Create watcher for config file and read initial configuration @@ -559,7 +559,7 @@ func (w *nfdWorker) advertiseFeatureLabels(labels Labels) error { labelReq := pb.SetLabelsRequest{Labels: labels, Features: source.GetAllFeatures(), NfdVersion: version.Get(), - NodeName: clientcommon.NodeName()} + NodeName: utils.NodeName()} cli, err := w.getGrpcClient() if err != nil { @@ -581,7 +581,7 @@ func (m *nfdWorker) updateNodeFeatureObject(labels Labels) error { if err != nil { return err } - nodename := clientcommon.NodeName() + nodename := utils.NodeName() namespace := m.kubernetesNamespace features := source.GetAllFeatures() diff --git a/pkg/nfd-master/nfd-master.go b/pkg/nfd-master/nfd-master.go index 7446471c2e..4d7c3d13c0 100644 --- a/pkg/nfd-master/nfd-master.go +++ b/pkg/nfd-master/nfd-master.go @@ -21,7 +21,6 @@ import ( "crypto/x509" "fmt" "net" - "os" "path" "regexp" "sort" @@ -101,7 +100,7 @@ type nfdMaster struct { // NewNfdMaster creates a new NfdMaster server instance. func NewNfdMaster(args *Args) (NfdMaster, error) { nfd := &nfdMaster{args: *args, - nodeName: os.Getenv("NODE_NAME"), + nodeName: utils.NodeName(), namespace: utils.GetKubernetesNamespace(), ready: make(chan bool, 1), stop: make(chan struct{}, 1), diff --git a/pkg/utils/kubernetes.go b/pkg/utils/kubernetes.go index 4dd6ba09fd..7601da0fc5 100644 --- a/pkg/utils/kubernetes.go +++ b/pkg/utils/kubernetes.go @@ -21,6 +21,16 @@ import ( "strings" ) +var nodeName string + +// NodeName returns the name of the k8s node we're running on. +func NodeName() string { + if nodeName == "" { + nodeName = os.Getenv("NODE_NAME") + } + return nodeName +} + // GetKubernetesNamespace returns the kubernetes namespace we're running under, // or an empty string if the namespace cannot be determined. func GetKubernetesNamespace() string { diff --git a/source/system/system.go b/source/system/system.go index 6945ef24d4..9399bd1d9e 100644 --- a/source/system/system.go +++ b/source/system/system.go @@ -82,7 +82,7 @@ func (s *systemSource) Discover() error { // Get node name s.features.Attributes[NameFeature] = nfdv1alpha1.NewAttributeFeatures(nil) - s.features.Attributes[NameFeature].Elements["nodename"] = os.Getenv("NODE_NAME") + s.features.Attributes[NameFeature].Elements["nodename"] = utils.NodeName() // Get os-release information release, err := parseOSRelease() From 0283f6870299c14408daca9f0d43f17c33b184cd Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Fri, 23 Dec 2022 10:02:52 +0200 Subject: [PATCH 77/87] topology-updater: move code Move and rename the Go package. It has nothing to do with NFD gRPC client anymore so move it out of the nfd-client package. --- cmd/nfd-topology-updater/main.go | 2 +- .../nfd-topology-updater.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename pkg/{nfd-client/topology-updater => nfd-topology-updater}/nfd-topology-updater.go (99%) diff --git a/cmd/nfd-topology-updater/main.go b/cmd/nfd-topology-updater/main.go index 0db443e27f..514c985530 100644 --- a/cmd/nfd-topology-updater/main.go +++ b/cmd/nfd-topology-updater/main.go @@ -26,7 +26,7 @@ import ( "k8s.io/klog/v2" kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" - topology "sigs.k8s.io/node-feature-discovery/pkg/nfd-client/topology-updater" + topology "sigs.k8s.io/node-feature-discovery/pkg/nfd-topology-updater" "sigs.k8s.io/node-feature-discovery/pkg/resourcemonitor" "sigs.k8s.io/node-feature-discovery/pkg/topologypolicy" "sigs.k8s.io/node-feature-discovery/pkg/utils" diff --git a/pkg/nfd-client/topology-updater/nfd-topology-updater.go b/pkg/nfd-topology-updater/nfd-topology-updater.go similarity index 99% rename from pkg/nfd-client/topology-updater/nfd-topology-updater.go rename to pkg/nfd-topology-updater/nfd-topology-updater.go index d3072bde23..a5b3e26393 100644 --- a/pkg/nfd-client/topology-updater/nfd-topology-updater.go +++ b/pkg/nfd-topology-updater/nfd-topology-updater.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package topologyupdater +package nfdtopologyupdater import ( "fmt" From 1026d91d12a35664059dc5c1ef410a74d31b0af6 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Fri, 23 Dec 2022 10:45:07 +0200 Subject: [PATCH 78/87] worker: move code Simplify code bu dropping the unnecessary base client package. --- cmd/nfd-worker/main.go | 2 +- pkg/nfd-client/base.go | 136 ------------------ .../nfd-worker-internal_test.go | 2 +- .../worker => nfd-worker}/nfd-worker.go | 107 +++++++++++--- .../worker => nfd-worker}/nfd-worker_test.go | 30 ++-- 5 files changed, 99 insertions(+), 178 deletions(-) delete mode 100644 pkg/nfd-client/base.go rename pkg/{nfd-client/worker => nfd-worker}/nfd-worker-internal_test.go (99%) rename pkg/{nfd-client/worker => nfd-worker}/nfd-worker.go (86%) rename pkg/{nfd-client/worker => nfd-worker}/nfd-worker_test.go (78%) diff --git a/cmd/nfd-worker/main.go b/cmd/nfd-worker/main.go index 30e3290d17..2e6675e22f 100644 --- a/cmd/nfd-worker/main.go +++ b/cmd/nfd-worker/main.go @@ -24,7 +24,7 @@ import ( "k8s.io/klog/v2" - "sigs.k8s.io/node-feature-discovery/pkg/nfd-client/worker" + worker "sigs.k8s.io/node-feature-discovery/pkg/nfd-worker" "sigs.k8s.io/node-feature-discovery/pkg/utils" "sigs.k8s.io/node-feature-discovery/pkg/version" ) diff --git a/pkg/nfd-client/base.go b/pkg/nfd-client/base.go deleted file mode 100644 index e8f53ed059..0000000000 --- a/pkg/nfd-client/base.go +++ /dev/null @@ -1,136 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package nfdclient - -import ( - "crypto/tls" - "crypto/x509" - "fmt" - "os" - "time" - - "golang.org/x/net/context" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/credentials/insecure" - "k8s.io/klog/v2" - - "sigs.k8s.io/node-feature-discovery/pkg/utils" -) - -// NfdClient defines a common interface for NFD clients. -type NfdClient interface { - Run() error - Stop() -} - -// NfdBaseClient is a common base type for handling connections to nfd-master. -type NfdBaseClient struct { - args Args - clientConn *grpc.ClientConn -} - -// Args holds the common command line arguments for all nfd clients. -type Args struct { - CaFile string - CertFile string - KeyFile string - Kubeconfig string - Server string - ServerNameOverride string - - Klog map[string]*utils.KlogFlagVal -} - -// NewNfdBaseClient creates a new NfdBaseClient instance. -func NewNfdBaseClient(args *Args) (NfdBaseClient, error) { - nfd := NfdBaseClient{args: *args} - - // Check TLS related args - if args.CertFile != "" || args.KeyFile != "" || args.CaFile != "" { - if args.CertFile == "" { - return nfd, fmt.Errorf("-cert-file needs to be specified alongside -key-file and -ca-file") - } - if args.KeyFile == "" { - return nfd, fmt.Errorf("-key-file needs to be specified alongside -cert-file and -ca-file") - } - if args.CaFile == "" { - return nfd, fmt.Errorf("-ca-file needs to be specified alongside -cert-file and -key-file") - } - } - - return nfd, nil -} - -// ClientConn returns the grpc ClientConn object. -func (w *NfdBaseClient) ClientConn() *grpc.ClientConn { return w.clientConn } - -// Connect creates a gRPC client connection to nfd-master. -func (w *NfdBaseClient) Connect() error { - // Check that if a connection already exists - if w.clientConn != nil { - return fmt.Errorf("client connection already exists") - } - - // Dial and create a client - dialCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - dialOpts := []grpc.DialOption{grpc.WithBlock()} - if w.args.CaFile != "" || w.args.CertFile != "" || w.args.KeyFile != "" { - // Load client cert for client authentication - cert, err := tls.LoadX509KeyPair(w.args.CertFile, w.args.KeyFile) - if err != nil { - return fmt.Errorf("failed to load client certificate: %v", err) - } - // Load CA cert for server cert verification - caCert, err := os.ReadFile(w.args.CaFile) - if err != nil { - return fmt.Errorf("failed to read root certificate file: %v", err) - } - caPool := x509.NewCertPool() - if ok := caPool.AppendCertsFromPEM(caCert); !ok { - return fmt.Errorf("failed to add certificate from '%s'", w.args.CaFile) - } - // Create TLS config - tlsConfig := &tls.Config{ - Certificates: []tls.Certificate{cert}, - RootCAs: caPool, - ServerName: w.args.ServerNameOverride, - MinVersion: tls.VersionTLS13, - } - dialOpts = append(dialOpts, grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))) - } else { - dialOpts = append(dialOpts, grpc.WithTransportCredentials(insecure.NewCredentials())) - } - klog.Infof("connecting to nfd-master at %s ...", w.args.Server) - conn, err := grpc.DialContext(dialCtx, w.args.Server, dialOpts...) - if err != nil { - return err - } - w.clientConn = conn - - return nil -} - -// Disconnect closes the connection to NFD master -func (w *NfdBaseClient) Disconnect() { - if w.clientConn != nil { - klog.Infof("closing connection to nfd-master ...") - w.clientConn.Close() - } - w.clientConn = nil -} diff --git a/pkg/nfd-client/worker/nfd-worker-internal_test.go b/pkg/nfd-worker/nfd-worker-internal_test.go similarity index 99% rename from pkg/nfd-client/worker/nfd-worker-internal_test.go rename to pkg/nfd-worker/nfd-worker-internal_test.go index 7141977123..94852ee3aa 100644 --- a/pkg/nfd-client/worker/nfd-worker-internal_test.go +++ b/pkg/nfd-worker/nfd-worker-internal_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package worker +package nfdworker import ( "os" diff --git a/pkg/nfd-client/worker/nfd-worker.go b/pkg/nfd-worker/nfd-worker.go similarity index 86% rename from pkg/nfd-client/worker/nfd-worker.go rename to pkg/nfd-worker/nfd-worker.go index 1f7443cf0e..304eeb6983 100644 --- a/pkg/nfd-client/worker/nfd-worker.go +++ b/pkg/nfd-worker/nfd-worker.go @@ -14,9 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ -package worker +package nfdworker import ( + "crypto/tls" + "crypto/x509" "encoding/json" "fmt" "os" @@ -27,6 +29,9 @@ import ( "time" "golang.org/x/net/context" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/klog/v2" @@ -38,7 +43,6 @@ import ( nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/pkg/apis/nfd/v1alpha1" nfdclient "sigs.k8s.io/node-feature-discovery/pkg/generated/clientset/versioned" pb "sigs.k8s.io/node-feature-discovery/pkg/labeler" - clientcommon "sigs.k8s.io/node-feature-discovery/pkg/nfd-client" "sigs.k8s.io/node-feature-discovery/pkg/utils" "sigs.k8s.io/node-feature-discovery/pkg/version" "sigs.k8s.io/node-feature-discovery/source" @@ -57,6 +61,12 @@ import ( _ "sigs.k8s.io/node-feature-discovery/source/usb" ) +// NfdWorker is the interface for nfd-worker daemon +type NfdWorker interface { + Run() error + Stop() +} + // NFDConfig contains the configuration settings of NfdWorker. type NFDConfig struct { Core coreConfig @@ -80,14 +90,18 @@ type Labels map[string]string // Args are the command line arguments of NfdWorker. type Args struct { - clientcommon.Args - + CaFile string + CertFile string ConfigFile string EnableNodeFeatureApi bool + KeyFile string + Klog map[string]*utils.KlogFlagVal + Kubeconfig string Oneshot bool Options string + Server string + ServerNameOverride string - Klog map[string]*utils.KlogFlagVal Overrides ConfigOverrideArgs } @@ -100,10 +114,9 @@ type ConfigOverrideArgs struct { } type nfdWorker struct { - clientcommon.NfdBaseClient - args Args certWatch *utils.FsWatcher + clientConn *grpc.ClientConn configFilePath string config *NFDConfig kubernetesNamespace string @@ -119,21 +132,27 @@ type duration struct { } // NewNfdWorker creates new NfdWorker instance. -func NewNfdWorker(args *Args) (clientcommon.NfdClient, error) { - base, err := clientcommon.NewNfdBaseClient(&args.Args) - if err != nil { - return nil, err - } - +func NewNfdWorker(args *Args) (NfdWorker, error) { nfd := &nfdWorker{ - NfdBaseClient: base, - args: *args, config: &NFDConfig{}, kubernetesNamespace: utils.GetKubernetesNamespace(), stop: make(chan struct{}, 1), } + // Check TLS related args + if args.CertFile != "" || args.KeyFile != "" || args.CaFile != "" { + if args.CertFile == "" { + return nfd, fmt.Errorf("-cert-file needs to be specified alongside -key-file and -ca-file") + } + if args.KeyFile == "" { + return nfd, fmt.Errorf("-key-file needs to be specified alongside -cert-file and -ca-file") + } + if args.CaFile == "" { + return nfd, fmt.Errorf("-ca-file needs to be specified alongside -cert-file and -key-file") + } + } + if args.ConfigFile != "" { nfd.configFilePath = filepath.Clean(args.ConfigFile) } @@ -175,7 +194,7 @@ func (w *nfdWorker) Run() error { return err } - defer w.GrpcDisconnect() + defer w.grpcDisconnect() labelTrigger := time.After(0) for { @@ -214,7 +233,7 @@ func (w *nfdWorker) Run() error { } // Manage connection to master if w.config.Core.NoPublish || !w.args.EnableNodeFeatureApi { - w.GrpcDisconnect() + w.grpcDisconnect() } // Always re-label after a re-config event. This way the new config @@ -223,7 +242,7 @@ func (w *nfdWorker) Run() error { case <-w.certWatch.Events: klog.Infof("TLS certificate update, renewing connection to nfd-master") - w.GrpcDisconnect() + w.grpcDisconnect() case <-w.stop: klog.Infof("shutting down nfd-worker") @@ -249,18 +268,60 @@ func (w *nfdWorker) getGrpcClient() (pb.LabelerClient, error) { return w.grpcClient, nil } - if err := w.NfdBaseClient.Connect(); err != nil { + // Check that if a connection already exists + if w.clientConn != nil { + return nil, fmt.Errorf("client connection already exists") + } + + // Dial and create a client + dialCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + dialOpts := []grpc.DialOption{grpc.WithBlock()} + if w.args.CaFile != "" || w.args.CertFile != "" || w.args.KeyFile != "" { + // Load client cert for client authentication + cert, err := tls.LoadX509KeyPair(w.args.CertFile, w.args.KeyFile) + if err != nil { + return nil, fmt.Errorf("failed to load client certificate: %v", err) + } + // Load CA cert for server cert verification + caCert, err := os.ReadFile(w.args.CaFile) + if err != nil { + return nil, fmt.Errorf("failed to read root certificate file: %v", err) + } + caPool := x509.NewCertPool() + if ok := caPool.AppendCertsFromPEM(caCert); !ok { + return nil, fmt.Errorf("failed to add certificate from '%s'", w.args.CaFile) + } + // Create TLS config + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{cert}, + RootCAs: caPool, + ServerName: w.args.ServerNameOverride, + MinVersion: tls.VersionTLS13, + } + dialOpts = append(dialOpts, grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))) + } else { + dialOpts = append(dialOpts, grpc.WithTransportCredentials(insecure.NewCredentials())) + } + klog.Infof("connecting to nfd-master at %s ...", w.args.Server) + conn, err := grpc.DialContext(dialCtx, w.args.Server, dialOpts...) + if err != nil { return nil, err } + w.clientConn = conn - w.grpcClient = pb.NewLabelerClient(w.ClientConn()) + w.grpcClient = pb.NewLabelerClient(w.clientConn) return w.grpcClient, nil } -// GrpcDisconnect closes the gRPC connection to NFD master -func (w *nfdWorker) GrpcDisconnect() { - w.NfdBaseClient.Disconnect() +// grpcDisconnect closes the gRPC connection to NFD master +func (w *nfdWorker) grpcDisconnect() { + if w.clientConn != nil { + klog.Infof("closing connection to nfd-master ...") + w.clientConn.Close() + } + w.clientConn = nil w.grpcClient = nil } func (c *coreConfig) sanitize() { diff --git a/pkg/nfd-client/worker/nfd-worker_test.go b/pkg/nfd-worker/nfd-worker_test.go similarity index 78% rename from pkg/nfd-client/worker/nfd-worker_test.go rename to pkg/nfd-worker/nfd-worker_test.go index adf2b84f19..b80aa27b32 100644 --- a/pkg/nfd-client/worker/nfd-worker_test.go +++ b/pkg/nfd-worker/nfd-worker_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package worker_test +package nfdworker_test import ( "fmt" @@ -25,9 +25,8 @@ import ( . "github.com/smartystreets/goconvey/convey" - nfdclient "sigs.k8s.io/node-feature-discovery/pkg/nfd-client" - "sigs.k8s.io/node-feature-discovery/pkg/nfd-client/worker" master "sigs.k8s.io/node-feature-discovery/pkg/nfd-master" + worker "sigs.k8s.io/node-feature-discovery/pkg/nfd-worker" "sigs.k8s.io/node-feature-discovery/pkg/utils" "sigs.k8s.io/node-feature-discovery/test/data" ) @@ -76,9 +75,9 @@ func teardownTest(ctx testContext) { func TestNewNfdWorker(t *testing.T) { Convey("When initializing new NfdWorker instance", t, func() { Convey("When one of -cert-file, -key-file or -ca-file is missing", func() { - _, err := worker.NewNfdWorker(&worker.Args{Args: nfdclient.Args{CertFile: "crt", KeyFile: "key"}}) - _, err2 := worker.NewNfdWorker(&worker.Args{Args: nfdclient.Args{KeyFile: "key", CaFile: "ca"}}) - _, err3 := worker.NewNfdWorker(&worker.Args{Args: nfdclient.Args{CertFile: "crt", CaFile: "ca"}}) + _, err := worker.NewNfdWorker(&worker.Args{CertFile: "crt", KeyFile: "key"}) + _, err2 := worker.NewNfdWorker(&worker.Args{KeyFile: "key", CaFile: "ca"}) + _, err3 := worker.NewNfdWorker(&worker.Args{CertFile: "crt", CaFile: "ca"}) Convey("An error should be returned", func() { So(err, ShouldNotBeNil) So(err2, ShouldNotBeNil) @@ -94,8 +93,7 @@ func TestRun(t *testing.T) { Convey("When running nfd-worker against nfd-master", t, func() { Convey("When publishing features from fake source", func() { args := &worker.Args{ - Args: nfdclient.Args{ - Server: "localhost:8192"}, + Server: "localhost:8192", Oneshot: true, Overrides: worker.ConfigOverrideArgs{LabelSources: &utils.StringSliceVal{"fake"}}, } @@ -120,15 +118,13 @@ func TestRunTls(t *testing.T) { Convey("When running nfd-worker against nfd-master with mutual TLS auth enabled", t, func() { Convey("When publishing features from fake source", func() { workerArgs := worker.Args{ - Args: nfdclient.Args{ - CaFile: data.FilePath("ca.crt"), - CertFile: data.FilePath("nfd-test-worker.crt"), - KeyFile: data.FilePath("nfd-test-worker.key"), - Server: "localhost:8192", - ServerNameOverride: "nfd-test-master", - }, - Oneshot: true, - Overrides: worker.ConfigOverrideArgs{LabelSources: &utils.StringSliceVal{"fake"}}, + CaFile: data.FilePath("ca.crt"), + CertFile: data.FilePath("nfd-test-worker.crt"), + KeyFile: data.FilePath("nfd-test-worker.key"), + Server: "localhost:8192", + ServerNameOverride: "nfd-test-master", + Oneshot: true, + Overrides: worker.ConfigOverrideArgs{LabelSources: &utils.StringSliceVal{"fake"}}, } w, _ := worker.NewNfdWorker(&workerArgs) err := w.Run() From 842153a9077166e187b75f31d2c96a21cf12bf1f Mon Sep 17 00:00:00 2001 From: "Muyassarov, Feruzjon" Date: Wed, 28 Dec 2022 14:00:04 +0200 Subject: [PATCH 79/87] Docs: mention tainting in the intro section Signed-off-by: Muyassarov, Feruzjon --- docs/get-started/introduction.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/get-started/introduction.md b/docs/get-started/introduction.md index a4719def6e..21211a934f 100644 --- a/docs/get-started/introduction.md +++ b/docs/get-started/introduction.md @@ -17,7 +17,7 @@ sort: 1 This software enables node feature discovery for Kubernetes. It detects hardware features available on each node in a Kubernetes cluster, and -advertises those features using node labels. +advertises those features using node labels and optionally node taints. NFD consists of three software components: From d9dc4b09d5737f83e035f0257da7a79ab06aee9e Mon Sep 17 00:00:00 2001 From: "Muyassarov, Feruzjon" Date: Thu, 29 Dec 2022 12:38:52 +0200 Subject: [PATCH 80/87] Bump cpuid to v2.2.3 Bump cpuid to v2.2.3 which adds support for detecting Intel Sierra Forest instructions like AVXIFMA, AVXNECONVERT, AVXVNNIINT8 and CMPCCXADD. Signed-off-by: Muyassarov, Feruzjon --- docs/usage/features.md | 4 ++++ go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/usage/features.md b/docs/usage/features.md index 56b61478ae..39ebb52333 100644 --- a/docs/usage/features.md +++ b/docs/usage/features.md @@ -97,6 +97,10 @@ configuration options for details. | AVX512VNNI | AVX-512 vector neural network instructions | AVX512VP2INTERSECT | AVX-512 intersect for D/Q | AVX512VPOPCNTDQ | AVX-512 vector population count doubleword and quadword +| AVXIFMA | AVX-IFMA instructions +| AVXNECONVERT | AVX-NE-CONVERT instructions +| AVXVNNIINT8 | AVX-VNNI-INT8 instructions +| CMPCCXADD | CMPCCXADD instructions | ENQCMD | Enqueue Command | GFNI | Galois Field New Instructions | HYPERVISOR | Running under hypervisor diff --git a/go.mod b/go.mod index bc723481c9..fb1e6d0bcc 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/google/go-cmp v0.5.9 github.com/jaypipes/ghw v0.8.1-0.20210827132705-c7224150a17e github.com/k8stopologyawareschedwg/noderesourcetopology-api v0.0.12 - github.com/klauspost/cpuid/v2 v2.2.2 + github.com/klauspost/cpuid/v2 v2.2.3 github.com/onsi/ginkgo/v2 v2.4.0 github.com/onsi/gomega v1.23.0 github.com/smartystreets/assertions v1.2.0 diff --git a/go.sum b/go.sum index 60f8d38c7d..c7b6635fda 100644 --- a/go.sum +++ b/go.sum @@ -426,8 +426,8 @@ github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwS github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/cpuid/v2 v2.2.2 h1:xPMwiykqNK9VK0NYC3+jTMYv9I6Vl3YdjZgPZKG3zO0= -github.com/klauspost/cpuid/v2 v2.2.2/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= +github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= From 099f52ca3683ded9fcf218e4c7b467b79b00fa19 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Thu, 22 Dec 2022 15:49:43 +0200 Subject: [PATCH 81/87] test/e2e: more comprehensive test for NodeFeature objects Test creation of multiple NodeFeature objects per node, mocking 3rd party extensions. --- test/e2e/data/nodefeature-1.yaml | 31 ++++++++ test/e2e/data/nodefeature-2.yaml | 8 ++ test/e2e/node_feature_discovery.go | 124 ++++++++++++++++++++++++++--- test/e2e/utils/crd.go | 51 ++++++++++++ 4 files changed, 202 insertions(+), 12 deletions(-) create mode 100644 test/e2e/data/nodefeature-1.yaml create mode 100644 test/e2e/data/nodefeature-2.yaml diff --git a/test/e2e/data/nodefeature-1.yaml b/test/e2e/data/nodefeature-1.yaml new file mode 100644 index 0000000000..4b3b965235 --- /dev/null +++ b/test/e2e/data/nodefeature-1.yaml @@ -0,0 +1,31 @@ +apiVersion: nfd.k8s-sigs.io/v1alpha1 +kind: NodeFeature +metadata: + # This name should ensure that it's processed later than that from nfd-worker + name: zzz-e2e-features-1 +spec: + # Features for NodeFeatureRule matching + features: + flags: + e2e.flags: + elements: + flag_1: {} + flag_2: {} + attributes: + # Override features from the fake sources + fake.attribute: + elements: + attr_2: "true" + instances: + # Append to features from the fake sources + fake.instance: + elements: + - attributes: + attr_1: "true" + attr_2: "9" + # Labels to be created + labels: + e2e-nodefeature-test-1: "obj-1" + e2e-nodefeature-test-2: "obj-1" + # Override feature from nfd-worker + fake-fakefeature3: "overridden" diff --git a/test/e2e/data/nodefeature-2.yaml b/test/e2e/data/nodefeature-2.yaml new file mode 100644 index 0000000000..0696902d3f --- /dev/null +++ b/test/e2e/data/nodefeature-2.yaml @@ -0,0 +1,8 @@ +apiVersion: nfd.k8s-sigs.io/v1alpha1 +kind: NodeFeature +metadata: + name: zzz-e2e-features-2 +spec: + labels: + e2e-nodefeature-test-1: "overridden-from-obj-2" + e2e-nodefeature-test-3: "obj-2" diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery.go index 5ba26404a8..2553e30eca 100644 --- a/test/e2e/node_feature_discovery.go +++ b/test/e2e/node_feature_discovery.go @@ -494,6 +494,94 @@ var _ = SIGDescribe("Node Feature Discovery", func() { }) }) + // + // Test NodeFeature + // + Context("and NodeFeature objects deployed", func() { + It("labels from the NodeFeature objects should be created", func() { + if !useNodeFeatureApi { + Skip("NodeFeature API not enabled") + } + + // We pick one node targeted for our NodeFeature objects + nodes, err := getNonControlPlaneNodes(f.ClientSet) + Expect(err).NotTo(HaveOccurred()) + + targetNodeName := nodes[0].Name + Expect(targetNodeName).ToNot(BeEmpty(), "No suitable worker node found") + + By("Creating NodeFeature object") + nodeFeatures, err := testutils.CreateOrUpdateNodeFeaturesFromFile(nfdClient, "nodefeature-1.yaml", f.Namespace.Name, targetNodeName) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying node labels from NodeFeature object #1") + expectedLabels := map[string]k8sLabels{ + targetNodeName: { + nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-1": "obj-1", + nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-2": "obj-1", + nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature3": "overridden", + }, + } + Expect(waitForNfdNodeLabels(f.ClientSet, expectedLabels)).NotTo(HaveOccurred()) + + By("Deleting NodeFeature object") + err = nfdClient.NfdV1alpha1().NodeFeatures(f.Namespace.Name).Delete(context.TODO(), nodeFeatures[0], metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying node labels from NodeFeature object were removed") + Expect(waitForNfdNodeLabels(f.ClientSet, nil)).NotTo(HaveOccurred()) + + By("Creating nfd-worker daemonset") + podSpecOpts := createPodSpecOpts( + testpod.SpecWithContainerImage(dockerImage), + testpod.SpecWithContainerExtraArgs("-label-sources=fake"), + ) + workerDS := testds.NFDWorker(podSpecOpts...) + workerDS, err = f.ClientSet.AppsV1().DaemonSets(f.Namespace.Name).Create(context.TODO(), workerDS, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for worker daemonset pods to be ready") + Expect(testpod.WaitForReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred()) + + By("Verifying node labels from nfd-worker") + expectedLabels = map[string]k8sLabels{ + "*": { + nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature1": "true", + nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature2": "true", + nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature3": "true", + }, + } + Expect(waitForNfdNodeLabels(f.ClientSet, expectedLabels)).NotTo(HaveOccurred()) + + By("Re-creating NodeFeature object") + _, err = testutils.CreateOrUpdateNodeFeaturesFromFile(nfdClient, "nodefeature-1.yaml", f.Namespace.Name, targetNodeName) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying node labels from NodeFeature object #1 are created") + expectedLabels[targetNodeName] = k8sLabels{ + nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-1": "obj-1", + nfdv1alpha1.FeatureLabelNs + "/e2e-nodefeature-test-2": "obj-1", + nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature1": "true", + nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature2": "true", + nfdv1alpha1.FeatureLabelNs + "/fake-fakefeature3": "overridden", + } + Expect(waitForNfdNodeLabels(f.ClientSet, expectedLabels)).NotTo(HaveOccurred()) + + By("Creating extra namespace") + extraNs, err := f.CreateNamespace("node-feature-discvery-extra-ns", nil) + Expect(err).NotTo(HaveOccurred()) + + By("Create NodeFeature object in the extra namespace") + _, err = testutils.CreateOrUpdateNodeFeaturesFromFile(nfdClient, "nodefeature-2.yaml", extraNs.Name, targetNodeName) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying node labels from NodeFeature object #2 are created") + expectedLabels[targetNodeName][nfdv1alpha1.FeatureLabelNs+"/e2e-nodefeature-test-1"] = "overridden-from-obj-2" + expectedLabels[targetNodeName][nfdv1alpha1.FeatureLabelNs+"/e2e-nodefeature-test-3"] = "obj-2" + Expect(waitForNfdNodeLabels(f.ClientSet, expectedLabels)).NotTo(HaveOccurred()) + }) + }) + // // Test NodeFeatureRule // @@ -522,10 +610,13 @@ core: By("Waiting for daemonset pods to be ready") Expect(testpod.WaitForReady(f.ClientSet, f.Namespace.Name, workerDS.Spec.Template.Labels["name"], 5)).NotTo(HaveOccurred()) - expected := map[string]string{ - "feature.node.kubernetes.io/e2e-flag-test-1": "true", - "feature.node.kubernetes.io/e2e-attribute-test-1": "true", - "feature.node.kubernetes.io/e2e-instance-test-1": "true"} + expected := map[string]k8sLabels{ + "*": { + nfdv1alpha1.FeatureLabelNs + "/e2e-flag-test-1": "true", + nfdv1alpha1.FeatureLabelNs + "/e2e-attribute-test-1": "true", + nfdv1alpha1.FeatureLabelNs + "/e2e-instance-test-1": "true", + }, + } By("Creating NodeFeatureRules #1") Expect(testutils.CreateNodeFeatureRulesFromFile(nfdClient, "nodefeaturerule-1.yaml")).NotTo(HaveOccurred()) @@ -537,9 +628,9 @@ core: Expect(testutils.CreateNodeFeatureRulesFromFile(nfdClient, "nodefeaturerule-2.yaml")).NotTo(HaveOccurred()) // Add features from NodeFeatureRule #2 - expected["feature.node.kubernetes.io/e2e-matchany-test-1"] = "true" - expected["feature.node.kubernetes.io/e2e-template-test-1-instance_1"] = "found" - expected["feature.node.kubernetes.io/e2e-template-test-1-instance_2"] = "found" + expected["*"][nfdv1alpha1.FeatureLabelNs+"/e2e-matchany-test-1"] = "true" + expected["*"][nfdv1alpha1.FeatureLabelNs+"/e2e-template-test-1-instance_1"] = "found" + expected["*"][nfdv1alpha1.FeatureLabelNs+"/e2e-template-test-1-instance_2"] = "found" By("Verifying node labels from NodeFeatureRules #1 and #2") Expect(waitForNfdNodeLabels(f.ClientSet, expected)).NotTo(HaveOccurred()) @@ -636,8 +727,10 @@ func waitForNfdNodeAnnotations(cli clientset.Interface, expected map[string]stri return err } +type k8sLabels map[string]string + // waitForNfdNodeLabels waits for node to be labeled as expected. -func waitForNfdNodeLabels(cli clientset.Interface, expected map[string]string) error { +func waitForNfdNodeLabels(cli clientset.Interface, expected map[string]k8sLabels) error { poll := func() error { nodes, err := getNonControlPlaneNodes(cli) if err != nil { @@ -645,8 +738,15 @@ func waitForNfdNodeLabels(cli clientset.Interface, expected map[string]string) e } for _, node := range nodes { labels := nfdLabels(node.Labels) - if !cmp.Equal(expected, labels) { - return fmt.Errorf("node %q labels do not match expected, diff (expected vs. received): %s", node.Name, cmp.Diff(expected, labels)) + nodeExpected, ok := expected[node.Name] + if !ok { + nodeExpected = k8sLabels{} + if defaultExpected, ok := expected["*"]; ok { + nodeExpected = defaultExpected + } + } + if !cmp.Equal(nodeExpected, labels) { + return fmt.Errorf("node %q labels do not match expected, diff (expected vs. received): %s", node.Name, cmp.Diff(nodeExpected, labels)) } } return nil @@ -733,8 +833,8 @@ func getNonControlPlaneNodes(cli clientset.Interface) ([]corev1.Node, error) { } // nfdLabels gets labels that are in the nfd label namespace. -func nfdLabels(labels map[string]string) map[string]string { - ret := map[string]string{} +func nfdLabels(labels map[string]string) k8sLabels { + ret := k8sLabels{} for key, val := range labels { if strings.HasPrefix(key, nfdv1alpha1.FeatureLabelNs) { diff --git a/test/e2e/utils/crd.go b/test/e2e/utils/crd.go index 6d95d90668..6546f312af 100644 --- a/test/e2e/utils/crd.go +++ b/test/e2e/utils/crd.go @@ -59,6 +59,38 @@ func CreateNfdCRDs(cli extclient.Interface) ([]*apiextensionsv1.CustomResourceDe return newCRDs, nil } +// CreateOrUpdateNodeFeaturesFromFile creates/updates a NodeFeature object from a given file located under test data directory. +func CreateOrUpdateNodeFeaturesFromFile(cli nfdclientset.Interface, filename, namespace, nodename string) ([]string, error) { + objs, err := nodeFeaturesFromFile(filepath.Join(packagePath, "..", "data", filename)) + if err != nil { + return nil, err + } + + names := make([]string, len(objs)) + for i, obj := range objs { + obj.Namespace = namespace + if obj.Labels == nil { + obj.Labels = map[string]string{} + } + obj.Labels[nfdv1alpha1.NodeFeatureObjNodeNameLabel] = nodename + + if oldObj, err := cli.NfdV1alpha1().NodeFeatures(namespace).Get(context.TODO(), obj.Name, metav1.GetOptions{}); errors.IsNotFound(err) { + if _, err := cli.NfdV1alpha1().NodeFeatures(namespace).Create(context.TODO(), obj, metav1.CreateOptions{}); err != nil { + return names, fmt.Errorf("failed to create NodeFeature %w", err) + } + } else if err == nil { + obj.SetResourceVersion(oldObj.GetResourceVersion()) + if _, err = cli.NfdV1alpha1().NodeFeatures(namespace).Update(context.TODO(), obj, metav1.UpdateOptions{}); err != nil { + return names, fmt.Errorf("failed to update NodeFeature object: %w", err) + } + } else { + return names, fmt.Errorf("failed to get NodeFeature %w", err) + } + names[i] = obj.Name + } + return names, nil +} + // CreateNodeFeatureRuleFromFile creates a NodeFeatureRule object from a given file located under test data directory. func CreateNodeFeatureRulesFromFile(cli nfdclientset.Interface, filename string) error { objs, err := nodeFeatureRulesFromFile(filepath.Join(packagePath, "..", "data", filename)) @@ -139,6 +171,25 @@ func crdsFromFile(path string) ([]*apiextensionsv1.CustomResourceDefinition, err return crds, nil } +func nodeFeaturesFromFile(path string) ([]*nfdv1alpha1.NodeFeature, error) { + objs, err := apiObjsFromFile(path, nfdscheme.Codecs.UniversalDeserializer()) + if err != nil { + return nil, err + } + + crs := make([]*nfdv1alpha1.NodeFeature, len(objs)) + + for i, obj := range objs { + var ok bool + crs[i], ok = obj.(*nfdv1alpha1.NodeFeature) + if !ok { + return nil, fmt.Errorf("unexpected type %t when reading %q", obj, path) + } + } + + return crs, nil +} + func nodeFeatureRulesFromFile(path string) ([]*nfdv1alpha1.NodeFeatureRule, error) { objs, err := apiObjsFromFile(path, nfdscheme.Codecs.UniversalDeserializer()) if err != nil { From 9356efe81158aa01c0486dc65277121d6449e30c Mon Sep 17 00:00:00 2001 From: PiotrProkop Date: Mon, 9 Jan 2023 13:15:59 +0100 Subject: [PATCH 82/87] Upgrade github.com/k8stopologyawareschedwg/noderesourcetopology-api to v0.0.13 Signed-off-by: PiotrProkop --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fb1e6d0bcc..d9a714f8bc 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/golang/protobuf v1.5.2 github.com/google/go-cmp v0.5.9 github.com/jaypipes/ghw v0.8.1-0.20210827132705-c7224150a17e - github.com/k8stopologyawareschedwg/noderesourcetopology-api v0.0.12 + github.com/k8stopologyawareschedwg/noderesourcetopology-api v0.0.13 github.com/klauspost/cpuid/v2 v2.2.3 github.com/onsi/ginkgo/v2 v2.4.0 github.com/onsi/gomega v1.23.0 diff --git a/go.sum b/go.sum index c7b6635fda..928bf981cc 100644 --- a/go.sum +++ b/go.sum @@ -420,8 +420,8 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/k8stopologyawareschedwg/noderesourcetopology-api v0.0.12 h1:NhXbOjO1st8hIcVpegr3zw/AGG12vs3z//tCDDcfPpE= -github.com/k8stopologyawareschedwg/noderesourcetopology-api v0.0.12/go.mod h1:AkACMQGiTgCt0lQw3m7TTU8PLH9lYKNK5e9DqFf5VuM= +github.com/k8stopologyawareschedwg/noderesourcetopology-api v0.0.13 h1:Y1RjPskyGMkVtNL8lq75bEdjqgq8gi+JJ1oWaz/mIJE= +github.com/k8stopologyawareschedwg/noderesourcetopology-api v0.0.13/go.mod h1:AkACMQGiTgCt0lQw3m7TTU8PLH9lYKNK5e9DqFf5VuM= github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI= github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= From 1bae2867e22b4089d2a642483dc9a7ca9da98213 Mon Sep 17 00:00:00 2001 From: PiotrProkop Date: Mon, 9 Jan 2023 13:16:23 +0100 Subject: [PATCH 83/87] Release `v0.0.13` of NodeResourceTopology API added missing TopologyManagerPolicy. Expose new policies: * RestrictedContainerLevel * RestrictedPodLevel * BestEffortContainerLevel * BestEffortPodLevel Signed-off-by: PiotrProkop --- pkg/topologypolicy/topology-policy.go | 39 ++++++--- pkg/topologypolicy/topology-policy_test.go | 99 ++++++++++++++++++++++ 2 files changed, 128 insertions(+), 10 deletions(-) create mode 100644 pkg/topologypolicy/topology-policy_test.go diff --git a/pkg/topologypolicy/topology-policy.go b/pkg/topologypolicy/topology-policy.go index 3a73653465..75331f24ed 100644 --- a/pkg/topologypolicy/topology-policy.go +++ b/pkg/topologypolicy/topology-policy.go @@ -24,20 +24,39 @@ import ( // DetectTopologyPolicy returns string type which present // both Topology manager policy and scope func DetectTopologyPolicy(policy string, scope string) v1alpha1.TopologyManagerPolicy { + switch scope { + case config.PodTopologyManagerScope: + return detectPolicyPodScope(policy) + case config.ContainerTopologyManagerScope: + return detectPolicyContainerScope(policy) + default: + return v1alpha1.None + } +} + +func detectPolicyPodScope(policy string) v1alpha1.TopologyManagerPolicy { + switch policy { + case config.SingleNumaNodeTopologyManagerPolicy: + return v1alpha1.SingleNUMANodePodLevel + case config.RestrictedTopologyManagerPolicy: + return v1alpha1.RestrictedPodLevel + case config.BestEffortTopologyManagerPolicy: + return v1alpha1.BestEffortPodLevel + case config.NoneTopologyManagerPolicy: + return v1alpha1.None + default: + return v1alpha1.None + } +} + +func detectPolicyContainerScope(policy string) v1alpha1.TopologyManagerPolicy { switch policy { case config.SingleNumaNodeTopologyManagerPolicy: - if scope == config.PodTopologyManagerScope { - return v1alpha1.SingleNUMANodePodLevel - } else if scope == config.ContainerTopologyManagerScope { - return v1alpha1.SingleNUMANodeContainerLevel - } else { - // default scope for single-numa-node - return v1alpha1.SingleNUMANodeContainerLevel - } + return v1alpha1.SingleNUMANodeContainerLevel case config.RestrictedTopologyManagerPolicy: - return v1alpha1.Restricted + return v1alpha1.RestrictedContainerLevel case config.BestEffortTopologyManagerPolicy: - return v1alpha1.BestEffort + return v1alpha1.BestEffortContainerLevel case config.NoneTopologyManagerPolicy: return v1alpha1.None default: diff --git a/pkg/topologypolicy/topology-policy_test.go b/pkg/topologypolicy/topology-policy_test.go new file mode 100644 index 0000000000..b766814a8c --- /dev/null +++ b/pkg/topologypolicy/topology-policy_test.go @@ -0,0 +1,99 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package topologypolicy + +import ( + "testing" + + v1alpha1 "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/apis/topology/v1alpha1" +) + +func TestDetectTopologyPolicy(t *testing.T) { + testCases := []struct { + scope string + policy string + expected v1alpha1.TopologyManagerPolicy + }{ + { + policy: "best-effort", + scope: "pod", + expected: v1alpha1.BestEffortPodLevel, + }, + { + policy: "best-effort", + scope: "container", + expected: v1alpha1.BestEffortContainerLevel, + }, + { + policy: "restricted", + scope: "container", + expected: v1alpha1.RestrictedContainerLevel, + }, + { + policy: "restricted", + scope: "pod", + expected: v1alpha1.RestrictedPodLevel, + }, + { + policy: "single-numa-node", + scope: "pod", + expected: v1alpha1.SingleNUMANodePodLevel, + }, + { + policy: "single-numa-node", + scope: "container", + expected: v1alpha1.SingleNUMANodeContainerLevel, + }, + { + policy: "none", + scope: "container", + expected: v1alpha1.None, + }, + { + policy: "none", + scope: "pod", + expected: v1alpha1.None, + }, + { + policy: "non-existent", + scope: "pod", + expected: v1alpha1.None, + }, + { + policy: "non-existent", + scope: "container", + expected: v1alpha1.None, + }, + { + policy: "single-numa-node", + scope: "non-existent", + expected: v1alpha1.None, + }, + { + policy: "single-numa-node", + scope: "non-existent", + expected: v1alpha1.None, + }, + } + + for _, tc := range testCases { + actual := DetectTopologyPolicy(tc.policy, tc.scope) + if actual != tc.expected { + t.Errorf("Expected TopologyPolicy to equal: %s not: %s", tc.expected, actual) + } + } +} From 59afae50ba3a479d1d192531ef1fa201d678ae70 Mon Sep 17 00:00:00 2001 From: PiotrProkop Date: Tue, 3 Jan 2023 13:23:46 +0100 Subject: [PATCH 84/87] Add NodeResourceTopology garbage collector NodeResourceTopology(aka NRT) custom resource is used to enable NUMA aware Scheduling in Kubernetes. As of now node-feature-discovery daemons are used to advertise those resources but there is no service responsible for removing obsolete objects(without corresponding Kubernetes node). This patch adds new daemon called nfd-topology-gc which removes old NRTs. Signed-off-by: PiotrProkop --- cmd/nfd-topology-gc/main.go | 88 +++++++ cmd/nfd-topology-gc/main_test.go | 41 +++ .../base/rbac-topology-gc/kustomization.yaml | 9 + .../topology-gc-clusterrole.yaml | 25 ++ .../topology-gc-clusterrolebinding.yaml | 12 + .../topology-gc-serviceaccount.yaml | 4 + .../base/topology-gc/kustomization.yaml | 7 + deployment/base/topology-gc/topology-gc.yaml | 23 ++ .../templates/_helpers.tpl | 11 + .../templates/clusterrole.yaml | 31 +++ .../templates/clusterrolebinding.yaml | 18 ++ .../templates/serviceaccount.yaml | 15 ++ .../templates/topology-gc.yaml | 64 +++++ .../helm/node-feature-discovery/values.yaml | 38 +++ .../kustomization.yaml | 2 + .../topologyupdater/kustomization.yaml | 2 + pkg/nfd-topology-gc/nfd-nrt-gc.go | 194 +++++++++++++++ pkg/nfd-topology-gc/nfd-nrt-gc_test.go | 234 ++++++++++++++++++ 18 files changed, 818 insertions(+) create mode 100644 cmd/nfd-topology-gc/main.go create mode 100644 cmd/nfd-topology-gc/main_test.go create mode 100644 deployment/base/rbac-topology-gc/kustomization.yaml create mode 100644 deployment/base/rbac-topology-gc/topology-gc-clusterrole.yaml create mode 100644 deployment/base/rbac-topology-gc/topology-gc-clusterrolebinding.yaml create mode 100644 deployment/base/rbac-topology-gc/topology-gc-serviceaccount.yaml create mode 100644 deployment/base/topology-gc/kustomization.yaml create mode 100644 deployment/base/topology-gc/topology-gc.yaml create mode 100644 deployment/helm/node-feature-discovery/templates/topology-gc.yaml create mode 100644 pkg/nfd-topology-gc/nfd-nrt-gc.go create mode 100644 pkg/nfd-topology-gc/nfd-nrt-gc_test.go diff --git a/cmd/nfd-topology-gc/main.go b/cmd/nfd-topology-gc/main.go new file mode 100644 index 0000000000..ac66bb3b03 --- /dev/null +++ b/cmd/nfd-topology-gc/main.go @@ -0,0 +1,88 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "fmt" + "os" + "time" + + "k8s.io/klog/v2" + + nfdtopologygarbagecollector "sigs.k8s.io/node-feature-discovery/pkg/nfd-topology-gc" + "sigs.k8s.io/node-feature-discovery/pkg/version" +) + +const ( + // ProgramName is the canonical name of this program + ProgramName = "nfd-topology-gc" +) + +func main() { + flags := flag.NewFlagSet(ProgramName, flag.ExitOnError) + + printVersion := flags.Bool("version", false, "Print version and exit.") + + args := parseArgs(flags, os.Args[1:]...) + + if *printVersion { + fmt.Println(ProgramName, version.Get()) + os.Exit(0) + } + + // Assert that the version is known + if version.Undefined() { + klog.Warningf("version not set! Set -ldflags \"-X sigs.k8s.io/node-feature-discovery/pkg/version.version=`git describe --tags --dirty --always`\" during build or run.") + } + + // Get new TopologyGC instance + gc, err := nfdtopologygarbagecollector.New(args) + if err != nil { + klog.Exit(err) + } + + if err = gc.Run(); err != nil { + klog.Exit(err) + } +} + +func parseArgs(flags *flag.FlagSet, osArgs ...string) *nfdtopologygarbagecollector.Args { + args := initFlags(flags) + + _ = flags.Parse(osArgs) + if len(flags.Args()) > 0 { + fmt.Fprintf(flags.Output(), "unknown command line argument: %s\n", flags.Args()[0]) + flags.Usage() + os.Exit(2) + } + + return args +} + +func initFlags(flagset *flag.FlagSet) *nfdtopologygarbagecollector.Args { + args := &nfdtopologygarbagecollector.Args{} + + flagset.DurationVar(&args.GCPeriod, "gc-interval", time.Duration(1)*time.Hour, + "Interval between which Garbage Collector will try to cleanup any missed but already obsolete NodeResourceTopology. [Default: 1h]") + flagset.StringVar(&args.Kubeconfig, "kubeconfig", "", + "Kubeconfig to use") + + klog.InitFlags(flagset) + + return args +} diff --git a/cmd/nfd-topology-gc/main_test.go b/cmd/nfd-topology-gc/main_test.go new file mode 100644 index 0000000000..541efa0488 --- /dev/null +++ b/cmd/nfd-topology-gc/main_test.go @@ -0,0 +1,41 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "testing" + "time" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestArgsParse(t *testing.T) { + Convey("When parsing command line arguments", t, func() { + flags := flag.NewFlagSet(ProgramName, flag.ExitOnError) + + Convey("When valid -gc-interval is specified", func() { + args := parseArgs(flags, + "-gc-interval=30s") + + Convey("args.GCPeriod is set to appropriate values", func() { + So(args.GCPeriod, ShouldEqual, 30*time.Second) + }) + }) + + }) +} diff --git a/deployment/base/rbac-topology-gc/kustomization.yaml b/deployment/base/rbac-topology-gc/kustomization.yaml new file mode 100644 index 0000000000..d0105ebc05 --- /dev/null +++ b/deployment/base/rbac-topology-gc/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: node-feature-discovery + +resources: +- topology-gc-clusterrole.yaml +- topology-gc-clusterrolebinding.yaml +- topology-gc-serviceaccount.yaml diff --git a/deployment/base/rbac-topology-gc/topology-gc-clusterrole.yaml b/deployment/base/rbac-topology-gc/topology-gc-clusterrole.yaml new file mode 100644 index 0000000000..c0f4314447 --- /dev/null +++ b/deployment/base/rbac-topology-gc/topology-gc-clusterrole.yaml @@ -0,0 +1,25 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nfd-topology-gc +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes/proxy + verbs: + - get +- apiGroups: + - topology.node.k8s.io + resources: + - noderesourcetopologies + verbs: + - delete + - list diff --git a/deployment/base/rbac-topology-gc/topology-gc-clusterrolebinding.yaml b/deployment/base/rbac-topology-gc/topology-gc-clusterrolebinding.yaml new file mode 100644 index 0000000000..b8615d63c0 --- /dev/null +++ b/deployment/base/rbac-topology-gc/topology-gc-clusterrolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: nfd-topology-gc +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nfd-topology-gc +subjects: +- kind: ServiceAccount + name: nfd-topology-gc + namespace: default diff --git a/deployment/base/rbac-topology-gc/topology-gc-serviceaccount.yaml b/deployment/base/rbac-topology-gc/topology-gc-serviceaccount.yaml new file mode 100644 index 0000000000..e56f7bbefd --- /dev/null +++ b/deployment/base/rbac-topology-gc/topology-gc-serviceaccount.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nfd-topology-gc diff --git a/deployment/base/topology-gc/kustomization.yaml b/deployment/base/topology-gc/kustomization.yaml new file mode 100644 index 0000000000..3d8da69b69 --- /dev/null +++ b/deployment/base/topology-gc/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: node-feature-discovery + +resources: +- topology-gc.yaml diff --git a/deployment/base/topology-gc/topology-gc.yaml b/deployment/base/topology-gc/topology-gc.yaml new file mode 100644 index 0000000000..07565e3a08 --- /dev/null +++ b/deployment/base/topology-gc/topology-gc.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: nfd + name: nfd-topology-gc +spec: + selector: + matchLabels: + app: nfd-topology-gc + template: + metadata: + labels: + app: nfd-topology-gc + spec: + dnsPolicy: ClusterFirstWithHostNet + serviceAccount: nfd-topology-gc + containers: + - name: nfd-topology-gc + image: gcr.io/k8s-staging-nfd/node-feature-discovery:master + imagePullPolicy: Always + command: + - "nfd-topology-gc" diff --git a/deployment/helm/node-feature-discovery/templates/_helpers.tpl b/deployment/helm/node-feature-discovery/templates/_helpers.tpl index 39c1e3df7f..5a0a5c97f7 100644 --- a/deployment/helm/node-feature-discovery/templates/_helpers.tpl +++ b/deployment/helm/node-feature-discovery/templates/_helpers.tpl @@ -94,3 +94,14 @@ Create the name of the service account which topologyUpdater will use {{ default "default" .Values.topologyUpdater.serviceAccount.name }} {{- end -}} {{- end -}} + +{{/* +Create the name of the service account which topologyGC will use +*/}} +{{- define "node-feature-discovery.topologyGC.serviceAccountName" -}} +{{- if .Values.topologyGC.serviceAccount.create -}} + {{ default (printf "%s-topology-gc" (include "node-feature-discovery.fullname" .)) .Values.topologyGC.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.topologyGC.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/deployment/helm/node-feature-discovery/templates/clusterrole.yaml b/deployment/helm/node-feature-discovery/templates/clusterrole.yaml index 3dd6f6f3b6..a282df376f 100644 --- a/deployment/helm/node-feature-discovery/templates/clusterrole.yaml +++ b/deployment/helm/node-feature-discovery/templates/clusterrole.yaml @@ -66,3 +66,34 @@ rules: - get - update {{- end }} + +--- +{{- if and .Values.topologyGC.enable .Values.topologyGC.rbac.create .Values.topologyUpdater.enable }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-topology-gc + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes/proxy + verbs: + - get +- apiGroups: + - topology.node.k8s.io + resources: + - noderesourcetopologies + verbs: + - delete + - list +{{- end }} diff --git a/deployment/helm/node-feature-discovery/templates/clusterrolebinding.yaml b/deployment/helm/node-feature-discovery/templates/clusterrolebinding.yaml index 5bceb41e76..227bce5e53 100644 --- a/deployment/helm/node-feature-discovery/templates/clusterrolebinding.yaml +++ b/deployment/helm/node-feature-discovery/templates/clusterrolebinding.yaml @@ -32,3 +32,21 @@ subjects: name: {{ include "node-feature-discovery.topologyUpdater.serviceAccountName" . }} namespace: {{ include "node-feature-discovery.namespace" . }} {{- end }} + +--- +{{- if and .Values.topologyGC.enable .Values.topologyGC.rbac.create .Values.topologyUpdater.enable }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-topology-gc + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "node-feature-discovery.fullname" . }}-topology-gc +subjects: +- kind: ServiceAccount + name: {{ .Values.topologyGC.serviceAccount.name | default "nfd-topology-gc" }} + namespace: {{ include "node-feature-discovery.namespace" . }} +{{- end }} diff --git a/deployment/helm/node-feature-discovery/templates/serviceaccount.yaml b/deployment/helm/node-feature-discovery/templates/serviceaccount.yaml index 883e5daabd..022961e454 100644 --- a/deployment/helm/node-feature-discovery/templates/serviceaccount.yaml +++ b/deployment/helm/node-feature-discovery/templates/serviceaccount.yaml @@ -27,6 +27,21 @@ metadata: {{- end }} {{- end }} +--- +{{- if and .Values.topologyGC.enable .Values.topologyGC.serviceAccount.create .Values.topologyUpdater.enable }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.topologyGC.serviceAccount.name | default "nfd-topology-gc" }} + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + {{- with .Values.topologyUpdater.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} + --- {{- if .Values.worker.serviceAccount.create }} apiVersion: v1 diff --git a/deployment/helm/node-feature-discovery/templates/topology-gc.yaml b/deployment/helm/node-feature-discovery/templates/topology-gc.yaml new file mode 100644 index 0000000000..642fec4559 --- /dev/null +++ b/deployment/helm/node-feature-discovery/templates/topology-gc.yaml @@ -0,0 +1,64 @@ +{{- if and .Values.topologyGC.enable .Values.topologyUpdater.enable -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-topology-gc + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + role: topology-gc +spec: + replicas: {{ .Values.topologyGC.replicaCount | default 1 }} + selector: + matchLabels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 6 }} + role: topology-gc + template: + metadata: + labels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 8 }} + role: topology-gc + annotations: + {{- toYaml .Values.topologyGC.annotations | nindent 8 }} + spec: + serviceAccountName: {{ .Values.topologyGC.serviceAccountName | default "nfd-topology-gc" }} + dnsPolicy: ClusterFirstWithHostNet + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.topologyGC.podSecurityContext | nindent 8 }} + containers: + - name: topology-gc + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: + - "nfd-topology-gc" + args: + {{- if .Values.topologyGC.interval | empty | not }} + - "-gc-interval={{ .Values.topologyGC.interval }}" + {{- end }} + resources: + {{- toYaml .Values.topologyGC.resources | nindent 12 }} + securityContext: + {{- toYaml .Values.topologyGC.securityContext | nindent 12 }} + + {{- with .Values.topologyGC.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.topologyGC.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.topologyGC.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/deployment/helm/node-feature-discovery/values.yaml b/deployment/helm/node-feature-discovery/values.yaml index 8531e963af..a30c42faa2 100644 --- a/deployment/helm/node-feature-discovery/values.yaml +++ b/deployment/helm/node-feature-discovery/values.yaml @@ -422,6 +422,44 @@ topologyUpdater: annotations: {} affinity: {} +topologyGC: + enable: true + replicaCount: 1 + + serviceAccount: + create: true + annotations: {} + name: + rbac: + create: true + + interval: 1h + + podSecurityContext: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ "ALL" ] + readOnlyRootFilesystem: true + runAsNonRoot: true + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + nodeSelector: {} + tolerations: [] + annotations: {} + affinity: {} + # Optionally use encryption for worker <--> master comms # TODO: verify hostname is not yet supported # diff --git a/deployment/overlays/master-worker-topologyupdater/kustomization.yaml b/deployment/overlays/master-worker-topologyupdater/kustomization.yaml index ffcbab6581..ebc3c1fc0a 100644 --- a/deployment/overlays/master-worker-topologyupdater/kustomization.yaml +++ b/deployment/overlays/master-worker-topologyupdater/kustomization.yaml @@ -6,11 +6,13 @@ namespace: node-feature-discovery bases: - ../../base/rbac - ../../base/rbac-topologyupdater +- ../../base/rbac-topology-gc - ../../base/nfd-crds - ../../base/master - ../../base/worker-daemonset - ../../base/noderesourcetopologies-crd - ../../base/topologyupdater-daemonset +- ../../base/topology-gc resources: - namespace.yaml diff --git a/deployment/overlays/topologyupdater/kustomization.yaml b/deployment/overlays/topologyupdater/kustomization.yaml index b344d993b8..964f990d0d 100644 --- a/deployment/overlays/topologyupdater/kustomization.yaml +++ b/deployment/overlays/topologyupdater/kustomization.yaml @@ -5,8 +5,10 @@ namespace: node-feature-discovery bases: - ../../base/rbac-topologyupdater +- ../../base/rbac-topology-gc - ../../base/noderesourcetopologies-crd - ../../base/topologyupdater-daemonset +- ../../base/topology-gc resources: - namespace.yaml diff --git a/pkg/nfd-topology-gc/nfd-nrt-gc.go b/pkg/nfd-topology-gc/nfd-nrt-gc.go new file mode 100644 index 0000000000..e59f597bd3 --- /dev/null +++ b/pkg/nfd-topology-gc/nfd-nrt-gc.go @@ -0,0 +1,194 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nfdtopologygarbagecollector + +import ( + "context" + "time" + + topologyclientset "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/generated/clientset/versioned" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + + "sigs.k8s.io/node-feature-discovery/pkg/apihelper" +) + +// Args are the command line arguments +type Args struct { + GCPeriod time.Duration + + Kubeconfig string +} + +type TopologyGC interface { + Run() error + Stop() +} + +type topologyGC struct { + stopChan chan struct{} + topoClient topologyclientset.Interface + gcPeriod time.Duration + factory informers.SharedInformerFactory +} + +func New(args *Args) (TopologyGC, error) { + kubeconfig, err := apihelper.GetKubeconfig(args.Kubeconfig) + if err != nil { + return nil, err + } + + stop := make(chan struct{}) + + return newTopologyGC(kubeconfig, stop, args.GCPeriod) +} + +func newTopologyGC(config *restclient.Config, stop chan struct{}, gcPeriod time.Duration) (*topologyGC, error) { + helper := apihelper.K8sHelpers{Kubeconfig: config} + cli, err := helper.GetTopologyClient() + if err != nil { + return nil, err + } + + clientset := kubernetes.NewForConfigOrDie(config) + factory := informers.NewSharedInformerFactory(clientset, 5*time.Minute) + + return &topologyGC{ + topoClient: cli, + stopChan: stop, + gcPeriod: gcPeriod, + factory: factory, + }, nil +} + +func (n *topologyGC) deleteNRT(nodeName string) { + if err := n.topoClient.TopologyV1alpha1().NodeResourceTopologies().Delete(context.TODO(), nodeName, metav1.DeleteOptions{}); err != nil { + if errors.IsNotFound(err) { + klog.V(2).Infof("NodeResourceTopology for node %s not found, omitting deletion", nodeName) + return + } else { + klog.Warningf("failed to delete NodeResourceTopology for node %s: %s", nodeName, err.Error()) + return + } + } + klog.Infof("NodeResourceTopology for node %s has been deleted", nodeName) +} + +func (n *topologyGC) deleteNodeHandler(object interface{}) { + // handle a case when we are starting up and need to clear stale NRT resources + obj := object + if deletedFinalStateUnknown, ok := object.(cache.DeletedFinalStateUnknown); ok { + klog.V(2).Infof("found stale NodeResourceTopology for node: %s ", object) + obj = deletedFinalStateUnknown.Obj + } + + node, ok := obj.(*corev1.Node) + if !ok { + klog.Errorf("cannot convert %v to v1.Node", object) + return + } + + n.deleteNRT(node.GetName()) +} + +func (n *topologyGC) runGC() { + klog.Infof("Running GC") + objects := n.factory.Core().V1().Nodes().Informer().GetIndexer().List() + nodes := sets.NewString() + for _, object := range objects { + key, err := cache.MetaNamespaceKeyFunc(object) + if err != nil { + klog.Warningf("cannot create key for %v: %s", object, err.Error()) + continue + } + nodes.Insert(key) + } + + nrts, err := n.topoClient.TopologyV1alpha1().NodeResourceTopologies().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + klog.Warningf("cannot list NRTs %s", err.Error()) + return + } + + for _, nrt := range nrts.Items { + key, err := cache.MetaNamespaceKeyFunc(&nrt) + if err != nil { + klog.Warningf("cannot create key for %v: %s", nrt, err.Error()) + continue + } + if !nodes.Has(key) { + n.deleteNRT(key) + } + } +} + +// periodicGC runs garbage collector at every gcPeriod to make sure we haven't missed any node +func (n *topologyGC) periodicGC(gcPeriod time.Duration) { + gcTrigger := time.NewTicker(gcPeriod) + for { + select { + case <-gcTrigger.C: + n.runGC() + case <-n.stopChan: + klog.Infof("shutting down periodic Garbage Collector") + return + } + } +} + +func (n *topologyGC) run() error { + nodeInformer := n.factory.Core().V1().Nodes().Informer() + + if _, err := nodeInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + DeleteFunc: n.deleteNodeHandler, + }); err != nil { + return err + } + + // start informers + n.factory.Start(n.stopChan) + n.factory.WaitForCacheSync(n.stopChan) + + n.runGC() + + return nil +} + +// Run is a blocking function that removes stale NRT objects when Node is deleted and runs periodic GC to make sure any obsolete objects are removed +func (n *topologyGC) Run() error { + if err := n.run(); err != nil { + return err + } + // run periodic GC + n.periodicGC(n.gcPeriod) + + return nil +} + +func (n *topologyGC) Stop() { + select { + case n.stopChan <- struct{}{}: + default: + } +} diff --git a/pkg/nfd-topology-gc/nfd-nrt-gc_test.go b/pkg/nfd-topology-gc/nfd-nrt-gc_test.go new file mode 100644 index 0000000000..c343fab375 --- /dev/null +++ b/pkg/nfd-topology-gc/nfd-nrt-gc_test.go @@ -0,0 +1,234 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nfdtopologygarbagecollector + +import ( + "context" + "testing" + "time" + + nrtapi "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/apis/topology/v1alpha1" + v1alpha1 "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/apis/topology/v1alpha1" + faketopologyv1alpha1 "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/generated/clientset/versioned/fake" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + fakek8sclientset "k8s.io/client-go/kubernetes/fake" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestNRTGC(t *testing.T) { + Convey("When theres is old NRT ", t, func() { + k8sClient := fakek8sclientset.NewSimpleClientset() + + fakeClient := faketopologyv1alpha1.NewSimpleClientset(&nrtapi.NodeResourceTopology{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + }) + factory := informers.NewSharedInformerFactory(k8sClient, 5*time.Minute) + + stopChan := make(chan struct{}, 1) + + gc := &topologyGC{ + factory: factory, + topoClient: fakeClient, + stopChan: stopChan, + gcPeriod: 10 * time.Minute, + } + + err := gc.run() + So(err, ShouldBeNil) + + nrts, err := fakeClient.TopologyV1alpha1().NodeResourceTopologies().List(context.TODO(), metav1.ListOptions{}) + So(err, ShouldBeNil) + So(nrts.Items, ShouldHaveLength, 0) + + gc.Stop() + }) + Convey("When theres is one old NRT and one up to date", t, func() { + k8sClient := fakek8sclientset.NewSimpleClientset(&corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + }) + + fakeClient := faketopologyv1alpha1.NewSimpleClientset(&nrtapi.NodeResourceTopology{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + }, + &nrtapi.NodeResourceTopology{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node2", + }, + }, + ) + + stopChan := make(chan struct{}, 1) + + factory := informers.NewSharedInformerFactory(k8sClient, 5*time.Minute) + + gc := &topologyGC{ + factory: factory, + topoClient: fakeClient, + stopChan: stopChan, + gcPeriod: 10 * time.Minute, + } + + err := gc.run() + So(err, ShouldBeNil) + + nrts, err := fakeClient.TopologyV1alpha1().NodeResourceTopologies().List(context.TODO(), metav1.ListOptions{}) + So(err, ShouldBeNil) + So(nrts.Items, ShouldHaveLength, 1) + So(nrts.Items[0].GetName(), ShouldEqual, "node1") + + }) + Convey("Should react to delete event", t, func() { + k8sClient := fakek8sclientset.NewSimpleClientset( + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + }, + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node2", + }, + }, + ) + + fakeClient := faketopologyv1alpha1.NewSimpleClientset( + &nrtapi.NodeResourceTopology{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + }, + &nrtapi.NodeResourceTopology{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node2", + }, + }, + ) + + stopChan := make(chan struct{}, 1) + + factory := informers.NewSharedInformerFactory(k8sClient, 5*time.Minute) + gc := &topologyGC{ + factory: factory, + topoClient: fakeClient, + stopChan: stopChan, + gcPeriod: 10 * time.Minute, + } + + err := gc.run() + So(err, ShouldBeNil) + + nrts, err := fakeClient.TopologyV1alpha1().NodeResourceTopologies().List(context.TODO(), metav1.ListOptions{}) + So(err, ShouldBeNil) + + So(nrts.Items, ShouldHaveLength, 2) + + err = k8sClient.CoreV1().Nodes().Delete(context.TODO(), "node1", metav1.DeleteOptions{}) + So(err, ShouldBeNil) + // simple sleep with retry loop to make sure indexer will pick up event and trigger deleteNode Function + deleted := false + for i := 0; i < 5; i++ { + nrts, err := fakeClient.TopologyV1alpha1().NodeResourceTopologies().List(context.TODO(), metav1.ListOptions{}) + So(err, ShouldBeNil) + + if len(nrts.Items) == 1 { + deleted = true + break + } + time.Sleep(time.Second) + } + So(deleted, ShouldBeTrue) + }) + Convey("periodic GC should remove obsolete NRT", t, func() { + k8sClient := fakek8sclientset.NewSimpleClientset( + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + }, + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node2", + }, + }, + ) + + fakeClient := faketopologyv1alpha1.NewSimpleClientset( + &nrtapi.NodeResourceTopology{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + }, + &nrtapi.NodeResourceTopology{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node2", + }, + }, + ) + + stopChan := make(chan struct{}, 1) + + factory := informers.NewSharedInformerFactory(k8sClient, 5*time.Minute) + gc := &topologyGC{ + factory: factory, + topoClient: fakeClient, + stopChan: stopChan, + gcPeriod: time.Second, + } + + err := gc.run() + So(err, ShouldBeNil) + + nrts, err := fakeClient.TopologyV1alpha1().NodeResourceTopologies().List(context.TODO(), metav1.ListOptions{}) + So(err, ShouldBeNil) + + So(nrts.Items, ShouldHaveLength, 2) + + nrt := v1alpha1.NodeResourceTopology{ + ObjectMeta: metav1.ObjectMeta{ + Name: "not-existing", + }, + } + + go gc.periodicGC(time.Second) + + _, err = fakeClient.TopologyV1alpha1().NodeResourceTopologies().Create(context.TODO(), &nrt, metav1.CreateOptions{}) + So(err, ShouldBeNil) + // simple sleep with retry loop to make sure GC was triggered + deleted := false + for i := 0; i < 5; i++ { + nrts, err := fakeClient.TopologyV1alpha1().NodeResourceTopologies().List(context.TODO(), metav1.ListOptions{}) + So(err, ShouldBeNil) + + if len(nrts.Items) == 2 { + deleted = true + break + } + time.Sleep(2 * time.Second) + } + So(deleted, ShouldBeTrue) + }) + +} From 3143faf0ab42d8be7a0d063c1d8061c7aa43d57f Mon Sep 17 00:00:00 2001 From: PiotrProkop Date: Tue, 10 Jan 2023 17:50:31 +0100 Subject: [PATCH 85/87] Add documentation for topology garbage collector Signed-off-by: PiotrProkop --- docs/deployment/helm.md | 19 ++++++++ docs/get-started/introduction.md | 8 ++++ .../topology-gc-commandline-reference.md | 46 +++++++++++++++++++ docs/usage/custom-resources.md | 2 +- docs/usage/customization-guide.md | 2 +- docs/usage/examples-and-demos.md | 2 +- docs/usage/nfd-topology-gc.md | 29 ++++++++++++ 7 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 docs/reference/topology-gc-commandline-reference.md create mode 100644 docs/usage/nfd-topology-gc.md diff --git a/docs/deployment/helm.md b/docs/deployment/helm.md index f618d76151..17b2ad7e87 100644 --- a/docs/deployment/helm.md +++ b/docs/deployment/helm.md @@ -173,5 +173,24 @@ We have introduced the following Chart parameters. | `topologyUpdater.affinity` | dict | {} | Topology updater pod [affinity](https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes-using-node-affinity/) | | `topologyUpdater.config` | dict | | [configuration](../reference/topology-updater-configuration-reference) | +### Topology garbage collector parameters + +| Name | Type | Default | description | +|-----------------------------------------------|--------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `topologyGC.*` | dict | | NFD Topology Garbage Collector configuration | +| `topologyGC.enable` | bool | true | Specifies whether the NFD Topology Garbage Collector should be created | +| `topologyGC.serviceAccount.create` | bool | true | Specifies whether the service account for topology garbage collector should be created | +| `topologyGC.serviceAccount.annotations` | dict | {} | Annotations to add to the service account for topology garbage collector | +| `topologyGC.serviceAccount.name` | string | | The name of the service account for topology garbage collector to use. If not set and create is true, a name is generated using the fullname template and `-topology-gc` suffix | +| `topologyGC.rbac.create` | bool | false | Specifies whether to create [RBAC][rbac] configuration for topology garbage collector | +| `topologyGC.interval` | string | 1h | Time between periodic garbage collector runs | +| `topologyGC.podSecurityContext` | dict | {} | [PodSecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) holds pod-level security attributes and common container settings | +| `topologyGC.securityContext` | dict | {} | Container [security settings](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) | +| `topologyGC.resources` | dict | {} | Topology garbage collector pod [resources management](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) | +| `topologyGC.nodeSelector` | dict | {} | Topology garbage collector pod [node selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) | +| `topologyGC.tolerations` | dict | {} | Topology garbage collector pod [node tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) | +| `topologyGC.annotations` | dict | {} | Topology garbage collector pod [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) | +| `topologyGC.affinity` | dict | {} | Topology garbage collector pod [affinity](https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes-using-node-affinity/) | + [rbac]: https://kubernetes.io/docs/reference/access-authn-authz/rbac/ diff --git a/docs/get-started/introduction.md b/docs/get-started/introduction.md index 21211a934f..b70ec2193c 100644 --- a/docs/get-started/introduction.md +++ b/docs/get-started/introduction.md @@ -47,6 +47,14 @@ creates or updates a resource object specific to this node. One instance of nfd-topology-updater is supposed to be running on each node of the cluster. +## NFD-Topology-Garbage-Collector + +NFD-Topology-Garbage-Collector is a daemon responsible for cleaning obsolete +[NodeResourceTopology](../usage/custom-resources#noderesourcetopology) objects, +obsolete means that there is no corresponding worker node. + +One instance of nfd-topology-gc is supposed to be running in the cluster. + ## Feature Discovery Feature discovery is divided into domain-specific feature sources: diff --git a/docs/reference/topology-gc-commandline-reference.md b/docs/reference/topology-gc-commandline-reference.md new file mode 100644 index 0000000000..bfabb1f196 --- /dev/null +++ b/docs/reference/topology-gc-commandline-reference.md @@ -0,0 +1,46 @@ +--- +title: "Topology Garbage Collector Cmdline Reference" +layout: default +sort: 6 +--- + +# NFD-Topology-Garbage-Collector Commandline Flags + +{: .no_toc } + +## Table of Contents + +{: .no_toc .text-delta } + +1. TOC +{:toc} + +--- + +To quickly view available command line flags execute `nfd-topology-gc -help`. +In a docker container: + +```bash +docker run {{ site.container_image }} \ +nfd-topology-gc -help +``` + +### -h, -help + +Print usage and exit. + +### -version + +Print version and exit. + +### -gc-interval + +The `-gc-interval` specifies the interval between periodic garbage collector runs. + +Default: 1h + +Example: + +```bash +nfd-topology-gc -gc-interval=1h +``` diff --git a/docs/usage/custom-resources.md b/docs/usage/custom-resources.md index 1fd64d0ea0..10e75769be 100644 --- a/docs/usage/custom-resources.md +++ b/docs/usage/custom-resources.md @@ -1,7 +1,7 @@ --- title: "CRDs" layout: default -sort: 6 +sort: 7 --- # Custom Resources diff --git a/docs/usage/customization-guide.md b/docs/usage/customization-guide.md index a73c20d9b0..8356b8b218 100644 --- a/docs/usage/customization-guide.md +++ b/docs/usage/customization-guide.md @@ -1,7 +1,7 @@ --- title: "Customization guide" layout: default -sort: 7 +sort: 8 --- # Customization guide diff --git a/docs/usage/examples-and-demos.md b/docs/usage/examples-and-demos.md index 7d64926292..36936bf64f 100644 --- a/docs/usage/examples-and-demos.md +++ b/docs/usage/examples-and-demos.md @@ -1,7 +1,7 @@ --- title: "Examples and demos" layout: default -sort: 8 +sort: 9 --- # Examples and demos diff --git a/docs/usage/nfd-topology-gc.md b/docs/usage/nfd-topology-gc.md new file mode 100644 index 0000000000..3af95f06f4 --- /dev/null +++ b/docs/usage/nfd-topology-gc.md @@ -0,0 +1,29 @@ +--- +title: "NFD-Topology-Garbage-Collector" +layout: default +sort: 6 +--- + +# NFD-Topology-Garbage-Collector +{: .no_toc} + +--- + +NFD-Topology-Garbage-Collector is preferably run as a Kubernetes deployment +with one replica. It makes sure that all +[NodeResourceTopology](custom-resources#noderesourcetopology) +have corresponding worker nodes and removes stale objects for worker nodes +which are no longer part of Kubernetes cluster. + +This service watches for Node deletion events and removes NodeResourceTopology +objects upon them. It is also running periodically to make sure no event was +missed or NodeResourceTopology object was created without corresponding worker +node. The default garbage collector interval is set to 1h which is the value +when no -gc-interval is specified. + +## Topology-Garbage-Collector Configuration + +In Helm deployments, +(see [Topology Garbage Collector](../deployment/helm.md#topology-garbage-collector-parameters) +for parameters). NFD-Topology-Garbage-Collector will only be deployed when +topologyUpdater.enable is set to true. From d8981f892e779c1bf701a6fbaf8b85e2dd54cec2 Mon Sep 17 00:00:00 2001 From: Talor Itzhak Date: Wed, 11 Jan 2023 14:15:45 +0200 Subject: [PATCH 86/87] e2e: append `_test` suffix to test files This PR is a result of conversation started here: https://github.com/kubernetes-sigs/node-feature-discovery/pull/1028#issuecomment-1378634404 Signed-off-by: Talor Itzhak --- .../{node_feature_discovery.go => node_feature_discovery_test.go} | 0 test/e2e/{topology_updater.go => topology_updater_test.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename test/e2e/{node_feature_discovery.go => node_feature_discovery_test.go} (100%) rename test/e2e/{topology_updater.go => topology_updater_test.go} (100%) diff --git a/test/e2e/node_feature_discovery.go b/test/e2e/node_feature_discovery_test.go similarity index 100% rename from test/e2e/node_feature_discovery.go rename to test/e2e/node_feature_discovery_test.go diff --git a/test/e2e/topology_updater.go b/test/e2e/topology_updater_test.go similarity index 100% rename from test/e2e/topology_updater.go rename to test/e2e/topology_updater_test.go From 97ca4deabc228bfe0c24aabf600df8a7e60b179e Mon Sep 17 00:00:00 2001 From: Talor Itzhak Date: Wed, 11 Jan 2023 12:14:43 +0200 Subject: [PATCH 87/87] e2e: init docker image The docker image that used during e2e test composed of repo and tag flags that are passed to the test itself. The problem is that the docker image initialized before the flags are parsed. Hence, it will always contains the default flags value. Moving the variable into a separate function, fixing the issue. Also, moving the global variables to `e2e_test.go` since it commonly used by all tests. Signed-off-by: Talor Itzhak --- test/e2e/e2e_test.go | 11 +++++++++++ test/e2e/node_feature_discovery_test.go | 16 ++++++---------- test/e2e/topology_updater_test.go | 4 ++-- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 6b25ba2854..39a8da9280 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -18,6 +18,7 @@ package e2e import ( "flag" + "fmt" "math/rand" "os" "testing" @@ -29,6 +30,11 @@ import ( "k8s.io/kubernetes/test/e2e/framework/testfiles" ) +var ( + dockerRepo = flag.String("nfd.repo", "gcr.io/k8s-staging-nfd/node-feature-discovery", "Docker repository to fetch image from") + dockerTag = flag.String("nfd.tag", "master", "Docker tag to use") +) + // handleFlags sets up all flags and parses the command line. func handleFlags() { config.CopyFlags(config.Flags, flag.CommandLine) @@ -37,6 +43,11 @@ func handleFlags() { flag.Parse() } +// must be called after flags are parsed +func dockerImage() string { + return fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag) +} + func TestMain(m *testing.M) { // Register test flags, then parse flags. handleFlags() diff --git a/test/e2e/node_feature_discovery_test.go b/test/e2e/node_feature_discovery_test.go index 2553e30eca..30306f2a4f 100644 --- a/test/e2e/node_feature_discovery_test.go +++ b/test/e2e/node_feature_discovery_test.go @@ -18,7 +18,6 @@ package e2e import ( "context" - "flag" "fmt" "path/filepath" "strings" @@ -49,9 +48,6 @@ import ( ) var ( - dockerRepo = flag.String("nfd.repo", "gcr.io/k8s-staging-nfd/node-feature-discovery", "Docker repository to fetch image from") - dockerTag = flag.String("nfd.tag", "master", "Docker tag to use") - dockerImage = fmt.Sprintf("%s:%s", *dockerRepo, *dockerTag) testTolerations = []corev1.Toleration{ { Key: "nfd.node.kubernetes.io/fake-special-node", @@ -216,7 +212,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { // Launch nfd-master By("Creating nfd master pod and nfd-master service") podSpecOpts := createPodSpecOpts( - testpod.SpecWithContainerImage(dockerImage), + testpod.SpecWithContainerImage(dockerImage()), testpod.SpecWithTolerations(testTolerations), testpod.SpecWithContainerExtraArgs("-enable-taints"), ) @@ -265,7 +261,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { By("Creating a nfd worker pod") podSpecOpts := createPodSpecOpts( testpod.SpecWithRestartPolicy(corev1.RestartPolicyNever), - testpod.SpecWithContainerImage(dockerImage), + testpod.SpecWithContainerImage(dockerImage()), testpod.SpecWithContainerExtraArgs("-oneshot", "-label-sources=fake"), testpod.SpecWithTolerations(testTolerations), ) @@ -318,7 +314,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { By("Creating nfd-worker daemonset") podSpecOpts := createPodSpecOpts( - testpod.SpecWithContainerImage(dockerImage), + testpod.SpecWithContainerImage(dockerImage()), testpod.SpecWithTolerations(testTolerations), ) workerDS := testds.NFDWorker(podSpecOpts...) @@ -448,7 +444,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { By("Creating nfd-worker daemonset with configmap mounted") podSpecOpts := createPodSpecOpts( - testpod.SpecWithContainerImage(dockerImage), + testpod.SpecWithContainerImage(dockerImage()), testpod.SpecWithConfigMap(cm1.Name, filepath.Join(custom.Directory, "cm1")), testpod.SpecWithConfigMap(cm2.Name, filepath.Join(custom.Directory, "cm2")), testpod.SpecWithTolerations(testTolerations), @@ -533,7 +529,7 @@ var _ = SIGDescribe("Node Feature Discovery", func() { By("Creating nfd-worker daemonset") podSpecOpts := createPodSpecOpts( - testpod.SpecWithContainerImage(dockerImage), + testpod.SpecWithContainerImage(dockerImage()), testpod.SpecWithContainerExtraArgs("-label-sources=fake"), ) workerDS := testds.NFDWorker(podSpecOpts...) @@ -599,7 +595,7 @@ core: By("Creating nfd-worker daemonset") podSpecOpts := createPodSpecOpts( - testpod.SpecWithContainerImage(dockerImage), + testpod.SpecWithContainerImage(dockerImage()), testpod.SpecWithConfigMap(cm.Name, "/etc/kubernetes/node-feature-discovery"), testpod.SpecWithTolerations(testTolerations), ) diff --git a/test/e2e/topology_updater_test.go b/test/e2e/topology_updater_test.go index 8afd1399b2..d63bca073f 100644 --- a/test/e2e/topology_updater_test.go +++ b/test/e2e/topology_updater_test.go @@ -115,7 +115,7 @@ var _ = SIGDescribe("Node Feature Discovery topology updater", func() { kcfg := cfg.GetKubeletConfig() By(fmt.Sprintf("Using config (%#v)", kcfg)) - podSpecOpts := []testpod.SpecOption{testpod.SpecWithContainerImage(dockerImage)} + podSpecOpts := []testpod.SpecOption{testpod.SpecWithContainerImage(dockerImage())} topologyUpdaterDaemonSet = testds.NFDTopologyUpdater(kcfg, podSpecOpts...) }) @@ -281,7 +281,7 @@ excludeList: By(fmt.Sprintf("Using config (%#v)", kcfg)) podSpecOpts := []testpod.SpecOption{ - testpod.SpecWithContainerImage(dockerImage), + testpod.SpecWithContainerImage(dockerImage()), testpod.SpecWithConfigMap(cm.Name, "/etc/kubernetes/node-feature-discovery"), } topologyUpdaterDaemonSet = testds.NFDTopologyUpdater(kcfg, podSpecOpts...)