From fd786e2029d116a94bfb83f38aee407f235400aa Mon Sep 17 00:00:00 2001 From: Andrea Panattoni Date: Tue, 5 Dec 2023 17:44:44 +0100 Subject: [PATCH 01/21] operator: LeaderElectionReleaseOnCancel When manually restarting the operator, the leader election may take 5+ minutest to acquire the lease on startup: ``` I1205 16:06:02.101302 1 leaderelection.go:245] attempting to acquire leader lease openshift-sriov-network-operator/a56def2a.openshift.io... ... I1205 16:08:40.133558 1 leaderelection.go:255] successfully acquired lease openshift-sriov-network-operator/a56def2a.openshift.io ``` The manager's option `LeaderElectionReleaseOnCancel` would solve this problem, but it's not safe as the shutdown cleanup procedures (inhibiting webhooks and removing finalizers) would run without any leader guard. This commit moves the LeaderElection mechanism from the namespaced manager to a dedicated, no-op controller manager. This approach has been preferred to directly dealing with the LeaderElection API as: - It leverages library code that has been proved to be stable - It includes recording k8s Events about the Lease process - The election process must come after setting up the health probe. Doing it manually would involve handling the healthz endpoint as well. Signed-off-by: Andrea Panattoni --- main.go | 129 ++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 103 insertions(+), 26 deletions(-) diff --git a/main.go b/main.go index f04bbb7ca..8982fb464 100644 --- a/main.go +++ b/main.go @@ -100,22 +100,42 @@ func main() { le := leaderelection.GetLeaderElectionConfig(kubeClient, enableLeaderElection) + leaderElectionMgr, err := ctrl.NewManager(restConfig, ctrl.Options{ + Scheme: scheme, + HealthProbeBindAddress: probeAddr, + Metrics: server.Options{BindAddress: "0"}, + LeaderElection: enableLeaderElection, + LeaseDuration: &le.LeaseDuration, + LeaderElectionReleaseOnCancel: true, + RenewDeadline: &le.RenewDeadline, + RetryPeriod: &le.RetryPeriod, + LeaderElectionID: "a56def2a.openshift.io", + }) + if err != nil { + setupLog.Error(err, "unable to start leader election manager") + os.Exit(1) + } + + if err := leaderElectionMgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up health check") + os.Exit(1) + } + if err := leaderElectionMgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up ready check") + os.Exit(1) + } + mgr, err := ctrl.NewManager(restConfig, ctrl.Options{ - Scheme: scheme, - Metrics: server.Options{BindAddress: metricsAddr}, - WebhookServer: webhook.NewServer(webhook.Options{Port: 9443}), - HealthProbeBindAddress: probeAddr, - LeaderElection: enableLeaderElection, - LeaseDuration: &le.LeaseDuration, - RenewDeadline: &le.RenewDeadline, - RetryPeriod: &le.RetryPeriod, - LeaderElectionID: "a56def2a.openshift.io", - Cache: cache.Options{DefaultNamespaces: map[string]cache.Config{vars.Namespace: {}}}, + Scheme: scheme, + Metrics: server.Options{BindAddress: metricsAddr}, + WebhookServer: webhook.NewServer(webhook.Options{Port: 9443}), + Cache: cache.Options{DefaultNamespaces: map[string]cache.Config{vars.Namespace: {}}}, }) if err != nil { setupLog.Error(err, "unable to start manager") os.Exit(1) } + mgrGlobal, err := ctrl.NewManager(restConfig, ctrl.Options{ Scheme: scheme, Metrics: server.Options{BindAddress: "0"}, @@ -225,29 +245,86 @@ func main() { } // +kubebuilder:scaffold:builder - if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up health check") - os.Exit(1) - } - if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up ready check") + leaderElectionErr := make(chan error) + leaderElectionContext, cancelLeaderElection := context.WithCancel(context.Background()) + go func() { + setupLog.Info("starting leader election manager") + leaderElectionErr <- leaderElectionMgr.Start(leaderElectionContext) + }() + + select { + case <-leaderElectionMgr.Elected(): + case err := <-leaderElectionErr: + setupLog.Error(err, "Leader Election Manager error") os.Exit(1) } - stopCh := ctrl.SetupSignalHandler() + setupLog.Info("acquired lease") + + stopSignalCh := ctrl.SetupSignalHandler() + + globalManagerErr := make(chan error) + globalManagerCtx, globalManagerCancel := context.WithCancel(context.Background()) go func() { - if err := mgrGlobal.Start(stopCh); err != nil { - setupLog.Error(err, "Manager Global exited non-zero") - os.Exit(1) - } + setupLog.Info("starting global manager") + globalManagerErr <- mgrGlobal.Start(globalManagerCtx) }() - // Remove all finalizers after controller is shut down - defer utils.Shutdown() + namespacedManagerErr := make(chan error) + namespacedManagerCtx, namespacedManagerCancel := context.WithCancel(context.Background()) + go func() { + setupLog.Info("starting namespaced manager") + namespacedManagerErr <- mgr.Start(namespacedManagerCtx) + }() + + select { + // Wait for a stop signal + case <-stopSignalCh.Done(): + setupLog.Info("Stop signal received") + + globalManagerCancel() + namespacedManagerCancel() + <-globalManagerErr + <-namespacedManagerErr + + utils.Shutdown() + + cancelLeaderElection() + <-leaderElectionErr + + case err := <-leaderElectionErr: + setupLog.Error(err, "Leader Election Manager error") + globalManagerCancel() + namespacedManagerCancel() + <-globalManagerErr + <-namespacedManagerErr + + os.Exit(1) + + case err := <-globalManagerErr: + setupLog.Error(err, "Global Manager error") + + namespacedManagerCancel() + <-namespacedManagerErr + + utils.Shutdown() + + cancelLeaderElection() + <-leaderElectionErr + + os.Exit(1) + + case err := <-namespacedManagerErr: + setupLog.Error(err, "Namsepaced Manager error") + + globalManagerCancel() + <-globalManagerErr + + utils.Shutdown() + + cancelLeaderElection() + <-leaderElectionErr - setupLog.Info("starting manager") - if err := mgr.Start(stopCh); err != nil { - setupLog.Error(err, "problem running manager") os.Exit(1) } } From 8431672ae42143415e54d036d4e2ba98da713899 Mon Sep 17 00:00:00 2001 From: Andrea Panattoni Date: Wed, 28 Feb 2024 12:07:29 +0100 Subject: [PATCH 02/21] e2e: dump info on deploy-wait.sh error Signed-off-by: Andrea Panattoni --- hack/deploy-wait.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hack/deploy-wait.sh b/hack/deploy-wait.sh index d4e66281e..5a0af6023 100755 --- a/hack/deploy-wait.sh +++ b/hack/deploy-wait.sh @@ -20,6 +20,7 @@ done if ! $ready; then echo "Timed out waiting for features to be ready" - oc get nodes + kubectl get nodes + kubectl cluster-info dump -n ${NAMESPACE} exit 1 fi From 0ef001c56e999430bffa37b6c4b00a8923a77316 Mon Sep 17 00:00:00 2001 From: Andrea Panattoni Date: Tue, 27 Feb 2024 14:49:19 +0100 Subject: [PATCH 03/21] e2e: Ensure operator restart quickly Add CoordinationV1 to `test/util/clients.go` to make assertions on `coordination.k8s.io/Lease` objects. Add `OPERATOR_LEADER_ELECTION_ENABLE` environment variable to `deploy/operator.yaml` to let user enable leader election on the operator. Signed-off-by: Andrea Panattoni --- deploy/operator.yaml | 2 + deploy/role.yaml | 6 +++ .../templates/role.yaml | 6 +++ hack/env.sh | 1 + hack/run-e2e-conformance-virtual-ocp.sh | 1 + main.go | 3 +- pkg/consts/constants.go | 1 + test/conformance/tests/test_sriov_operator.go | 46 +++++++++++++++++-- test/util/client/clients.go | 3 ++ 9 files changed, 65 insertions(+), 4 deletions(-) diff --git a/deploy/operator.yaml b/deploy/operator.yaml index 069f2eff1..e76a09647 100644 --- a/deploy/operator.yaml +++ b/deploy/operator.yaml @@ -41,6 +41,8 @@ spec: image: $SRIOV_NETWORK_OPERATOR_IMAGE command: - sriov-network-operator + args: + - --leader-elect=$OPERATOR_LEADER_ELECTION_ENABLE resources: requests: cpu: 100m diff --git a/deploy/role.yaml b/deploy/role.yaml index 409286ab5..a24f13729 100644 --- a/deploy/role.yaml +++ b/deploy/role.yaml @@ -56,6 +56,12 @@ rules: - get - list - watch +- apiGroups: + - 'coordination.k8s.io' + resources: + - 'leases' + verbs: + - '*' --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role diff --git a/deployment/sriov-network-operator/templates/role.yaml b/deployment/sriov-network-operator/templates/role.yaml index 6058a86e1..29cf80cce 100644 --- a/deployment/sriov-network-operator/templates/role.yaml +++ b/deployment/sriov-network-operator/templates/role.yaml @@ -59,6 +59,12 @@ rules: - get - list - watch + - apiGroups: + - 'coordination.k8s.io' + resources: + - 'leases' + verbs: + - '*' --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role diff --git a/hack/env.sh b/hack/env.sh index 1dccb157e..bd2ba6111 100755 --- a/hack/env.sh +++ b/hack/env.sh @@ -30,3 +30,4 @@ export ADMISSION_CONTROLLERS_CERTIFICATES_CERT_MANAGER_ENABLED=${ADMISSION_CONTR export ADMISSION_CONTROLLERS_CERTIFICATES_OPERATOR_CA_CRT=${ADMISSION_CONTROLLERS_CERTIFICATES_OPERATOR_CA_CRT:-""} export ADMISSION_CONTROLLERS_CERTIFICATES_INJECTOR_CA_CRT=${ADMISSION_CONTROLLERS_CERTIFICATES_INJECTOR_CA_CRT:-""} export DEV_MODE=${DEV_MODE:-"FALSE"} +export OPERATOR_LEADER_ELECTION_ENABLE=${OPERATOR_LEADER_ELECTION_ENABLE:-"false"} diff --git a/hack/run-e2e-conformance-virtual-ocp.sh b/hack/run-e2e-conformance-virtual-ocp.sh index 03d788e0d..20859c669 100755 --- a/hack/run-e2e-conformance-virtual-ocp.sh +++ b/hack/run-e2e-conformance-virtual-ocp.sh @@ -189,6 +189,7 @@ export OPERATOR_EXEC=kubectl export CLUSTER_TYPE=openshift export DEV_MODE=TRUE export CLUSTER_HAS_EMULATED_PF=TRUE +export OPERATOR_LEADER_ELECTION_ENABLE=true export SRIOV_NETWORK_OPERATOR_IMAGE="$registry/$NAMESPACE/sriov-network-operator:latest" export SRIOV_NETWORK_CONFIG_DAEMON_IMAGE="$registry/$NAMESPACE/sriov-network-config-daemon:latest" diff --git a/main.go b/main.go index 8982fb464..bc1998bb6 100644 --- a/main.go +++ b/main.go @@ -49,6 +49,7 @@ import ( "github.com/k8snetworkplumbingwg/sriov-network-operator/controllers" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/leaderelection" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils" @@ -109,7 +110,7 @@ func main() { LeaderElectionReleaseOnCancel: true, RenewDeadline: &le.RenewDeadline, RetryPeriod: &le.RetryPeriod, - LeaderElectionID: "a56def2a.openshift.io", + LeaderElectionID: consts.LeaderElectionID, }) if err != nil { setupLog.Error(err, "unable to start leader election manager") diff --git a/pkg/consts/constants.go b/pkg/consts/constants.go index 86af30bef..04752643f 100644 --- a/pkg/consts/constants.go +++ b/pkg/consts/constants.go @@ -35,6 +35,7 @@ const ( ServiceAccount = "ServiceAccount" DPConfigFileName = "config.json" OVSHWOLMachineConfigNameSuffix = "ovs-hw-offload" + LeaderElectionID = "a56def2a.openshift.io" LinkTypeEthernet = "ether" LinkTypeInfiniband = "infiniband" diff --git a/test/conformance/tests/test_sriov_operator.go b/test/conformance/tests/test_sriov_operator.go index 9ad400cbc..c033060a7 100644 --- a/test/conformance/tests/test_sriov_operator.go +++ b/test/conformance/tests/test_sriov_operator.go @@ -28,6 +28,7 @@ import ( runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" sriovv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/clean" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/cluster" "github.com/k8snetworkplumbingwg/sriov-network-operator/test/util/discovery" @@ -276,6 +277,35 @@ var _ = Describe("[sriov] operator", func() { }, 3*time.Minute, 5*time.Second).Should(Succeed()) }) }) + + It("should gracefully restart quickly", func() { + // This test case ensure leader election process runs smoothly when the operator's pod is restarted. + oldLease, err := clients.CoordinationV1Interface.Leases(operatorNamespace).Get(context.Background(), consts.LeaderElectionID, metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { + Skip("Leader Election is not enabled on the cluster. Skipping") + } + Expect(err).ToNot(HaveOccurred()) + + oldOperatorPod := getOperatorPod() + + By("Delete the operator's pod") + deleteOperatorPod() + + By("Wait the new operator's pod to start") + Eventually(func(g Gomega) { + newOperatorPod := getOperatorPod() + Expect(newOperatorPod.Name).ToNot(Equal(oldOperatorPod.Name)) + Expect(newOperatorPod.Status.Phase).To(Equal(corev1.PodRunning)) + }, 45*time.Second, 5*time.Second) + + By("Assert the new operator's pod acquire the lease before 30 seconds") + Eventually(func(g Gomega) { + newLease, err := clients.CoordinationV1Interface.Leases(operatorNamespace).Get(context.Background(), consts.LeaderElectionID, metav1.GetOptions{}) + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(newLease.Spec.HolderIdentity).ToNot(Equal(oldLease.Spec.HolderIdentity)) + }, 30*time.Second, 5*time.Second).Should(Succeed()) + }) }) Describe("Generic SriovNetworkNodePolicy", func() { @@ -2627,14 +2657,17 @@ func getOperatorConfigLogLevel() int { return cfg.Spec.LogLevel } -func getOperatorLogs(since time.Time) []string { +func getOperatorPod() corev1.Pod { podList, err := clients.Pods(operatorNamespace).List(context.Background(), metav1.ListOptions{ LabelSelector: "name=sriov-network-operator", }) ExpectWithOffset(1, err).ToNot(HaveOccurred()) - ExpectWithOffset(1, podList.Items).To(HaveLen(1), "One operator pod expected") + ExpectWithOffset(1, podList.Items).ToNot(HaveLen(0), "At least one operator pod expected") + return podList.Items[0] +} - pod := podList.Items[0] +func getOperatorLogs(since time.Time) []string { + pod := getOperatorPod() logStart := metav1.NewTime(since) rawLogs, err := clients.Pods(pod.Namespace). GetLogs(pod.Name, &corev1.PodLogOptions{ @@ -2647,6 +2680,13 @@ func getOperatorLogs(since time.Time) []string { return strings.Split(string(rawLogs), "\n") } +func deleteOperatorPod() { + pod := getOperatorPod() + + err := clients.Pods(operatorNamespace).Delete(context.Background(), pod.Name, metav1.DeleteOptions{}) + ExpectWithOffset(1, err).ToNot(HaveOccurred()) +} + func assertObjectIsNotFound(name string, obj runtimeclient.Object) { Eventually(func() bool { err := clients.Get(context.Background(), runtimeclient.ObjectKey{Name: name, Namespace: operatorNamespace}, obj) diff --git a/test/util/client/clients.go b/test/util/client/clients.go index a96634c19..21eb12053 100644 --- a/test/util/client/clients.go +++ b/test/util/client/clients.go @@ -11,6 +11,7 @@ import ( discovery "k8s.io/client-go/discovery" clientgoscheme "k8s.io/client-go/kubernetes/scheme" appsv1client "k8s.io/client-go/kubernetes/typed/apps/v1" + coordinationv1 "k8s.io/client-go/kubernetes/typed/coordination/v1" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -37,6 +38,7 @@ type ClientSet struct { clientsriovv1.SriovnetworkV1Interface Config *rest.Config runtimeclient.Client + coordinationv1.CoordinationV1Interface } // New returns a *ClientBuilder with the given kubeconfig. @@ -67,6 +69,7 @@ func New(kubeconfig string) *ClientSet { clientSet.AppsV1Interface = appsv1client.NewForConfigOrDie(config) clientSet.DiscoveryInterface = discovery.NewDiscoveryClientForConfigOrDie(config) clientSet.SriovnetworkV1Interface = clientsriovv1.NewForConfigOrDie(config) + clientSet.CoordinationV1Interface = coordinationv1.NewForConfigOrDie(config) clientSet.Config = config crScheme := runtime.NewScheme() From cba73d362f7da0fd19ffd4f9a1d8d839c32ca008 Mon Sep 17 00:00:00 2001 From: Kristian-ZH Date: Tue, 26 Mar 2024 17:24:40 +0200 Subject: [PATCH 04/21] Update Helm chart version --- deployment/sriov-network-operator/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/sriov-network-operator/Chart.yaml b/deployment/sriov-network-operator/Chart.yaml index 8c7a62e95..4fffae29a 100644 --- a/deployment/sriov-network-operator/Chart.yaml +++ b/deployment/sriov-network-operator/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: sriov-network-operator -version: 0.1.0 +version: 1.2.0 kubeVersion: '>= 1.16.0-0' appVersion: 1.2.0 description: SR-IOV network operator configures and manages SR-IOV networks in the kubernetes cluster From 054078304409656331220f8881c2f41256e7832d Mon Sep 17 00:00:00 2001 From: Sebastian Sch Date: Wed, 10 Apr 2024 21:50:21 +0300 Subject: [PATCH 05/21] Fix crash panic in operator when policy has empty node selector The commit will fix a nil pointer exception in the operator when there is a policy with empty selector the update of the devicePlugin daemonSet will fail Signed-off-by: Sebastian Sch --- controllers/helper.go | 76 +++- controllers/helper_test.go | 330 ++++++++++++++++++ .../sriovnetworknodepolicy_controller.go | 64 ---- .../sriovnetworknodepolicy_controller_test.go | 114 ------ 4 files changed, 395 insertions(+), 189 deletions(-) create mode 100644 controllers/helper_test.go diff --git a/controllers/helper.go b/controllers/helper.go index 823e4a369..5d169b2c5 100644 --- a/controllers/helper.go +++ b/controllers/helper.go @@ -22,9 +22,12 @@ import ( "encoding/json" "fmt" "os" + "sort" "strings" + errs "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" uns "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -263,6 +266,66 @@ func syncDsObject(ctx context.Context, client k8sclient.Client, scheme *runtime. return nil } +func setDsNodeAffinity(pl *sriovnetworkv1.SriovNetworkNodePolicyList, ds *appsv1.DaemonSet) error { + terms := nodeSelectorTermsForPolicyList(pl.Items) + if len(terms) > 0 { + ds.Spec.Template.Spec.Affinity = &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: terms, + }, + }, + } + } + return nil +} + +func nodeSelectorTermsForPolicyList(policies []sriovnetworkv1.SriovNetworkNodePolicy) []corev1.NodeSelectorTerm { + terms := []corev1.NodeSelectorTerm{} + for _, p := range policies { + // Note(adrianc): default policy is deprecated and ignored. + if p.Name == constants.DefaultPolicyName { + continue + } + + if len(p.Spec.NodeSelector) == 0 { + continue + } + expressions := []corev1.NodeSelectorRequirement{} + for k, v := range p.Spec.NodeSelector { + exp := corev1.NodeSelectorRequirement{ + Operator: corev1.NodeSelectorOpIn, + Key: k, + Values: []string{v}, + } + expressions = append(expressions, exp) + } + // sorting is needed to keep the daemon spec stable. + // the items are popped in a random order from the map + sort.Slice(expressions, func(i, j int) bool { + return expressions[i].Key < expressions[j].Key + }) + nodeSelector := corev1.NodeSelectorTerm{ + MatchExpressions: expressions, + } + terms = append(terms, nodeSelector) + } + + return terms +} + +// renderDsForCR returns a busybox pod with the same name/namespace as the cr +func renderDsForCR(path string, data *render.RenderData) ([]*uns.Unstructured, error) { + logger := log.Log.WithName("renderDsForCR") + logger.V(1).Info("Start to render objects") + + objs, err := render.RenderDir(path, data) + if err != nil { + return nil, errs.Wrap(err, "failed to render SR-IOV Network Operator manifests") + } + return objs, nil +} + func syncDaemonSet(ctx context.Context, client k8sclient.Client, scheme *runtime.Scheme, dc *sriovnetworkv1.SriovOperatorConfig, pl *sriovnetworkv1.SriovNetworkNodePolicyList, in *appsv1.DaemonSet) error { logger := log.Log.WithName("syncDaemonSet") logger.V(1).Info("Start to sync DaemonSet", "Namespace", in.Namespace, "Name", in.Name) @@ -302,17 +365,8 @@ func syncDaemonSet(ctx context.Context, client k8sclient.Client, scheme *runtime if equality.Semantic.DeepEqual(in.OwnerReferences, ds.OwnerReferences) && equality.Semantic.DeepDerivative(in.Spec, ds.Spec) { - // DeepDerivative has issue detecting nodeAffinity change - // https://bugzilla.redhat.com/show_bug.cgi?id=1914066 - // DeepDerivative doesn't detect changes in containers args section - // This code should be fixed both with NodeAffinity comparation - if equality.Semantic.DeepEqual(in.Spec.Template.Spec.Affinity.NodeAffinity, - ds.Spec.Template.Spec.Affinity.NodeAffinity) && - equality.Semantic.DeepEqual(in.Spec.Template.Spec.Containers[0].Args, - ds.Spec.Template.Spec.Containers[0].Args) { - logger.V(1).Info("Daemonset spec did not change, not updating") - return nil - } + logger.V(1).Info("Daemonset spec did not change, not updating") + return nil } err = client.Update(ctx, in) if err != nil { diff --git a/controllers/helper_test.go b/controllers/helper_test.go new file mode 100644 index 000000000..d998cf0da --- /dev/null +++ b/controllers/helper_test.go @@ -0,0 +1,330 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "sync" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/google/go-cmp/cmp" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + controllerruntime "sigs.k8s.io/controller-runtime" + + sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" +) + +func TestNodeSelectorMerge(t *testing.T) { + table := []struct { + tname string + policies []sriovnetworkv1.SriovNetworkNodePolicy + expected []corev1.NodeSelectorTerm + }{ + { + tname: "testoneselector", + policies: []sriovnetworkv1.SriovNetworkNodePolicy{ + { + Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ + NodeSelector: map[string]string{ + "foo": "bar", + }, + }, + }, + { + Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ + NodeSelector: map[string]string{ + "bb": "cc", + }, + }, + }, + }, + expected: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Operator: corev1.NodeSelectorOpIn, + Key: "foo", + Values: []string{"bar"}, + }, + }, + }, + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Operator: corev1.NodeSelectorOpIn, + Key: "bb", + Values: []string{"cc"}, + }, + }, + }, + }, + }, + { + tname: "testtwoselectors", + policies: []sriovnetworkv1.SriovNetworkNodePolicy{ + { + Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ + NodeSelector: map[string]string{ + "foo": "bar", + "foo1": "bar1", + }, + }, + }, + { + Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ + NodeSelector: map[string]string{ + "bb": "cc", + "bb1": "cc1", + "bb2": "cc2", + }, + }, + }, + }, + expected: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Operator: corev1.NodeSelectorOpIn, + Key: "foo", + Values: []string{"bar"}, + }, + { + Operator: corev1.NodeSelectorOpIn, + Key: "foo1", + Values: []string{"bar1"}, + }, + }, + }, + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Operator: corev1.NodeSelectorOpIn, + Key: "bb", + Values: []string{"cc"}, + }, + { + Operator: corev1.NodeSelectorOpIn, + Key: "bb1", + Values: []string{"cc1"}, + }, + { + Operator: corev1.NodeSelectorOpIn, + Key: "bb2", + Values: []string{"cc2"}, + }, + }, + }, + }, + }, + { + tname: "testemptyselector", + policies: []sriovnetworkv1.SriovNetworkNodePolicy{ + { + Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ + NodeSelector: map[string]string{}, + }, + }, + }, + expected: []corev1.NodeSelectorTerm{}, + }, + } + + for _, tc := range table { + t.Run(tc.tname, func(t *testing.T) { + selectors := nodeSelectorTermsForPolicyList(tc.policies) + if !cmp.Equal(selectors, tc.expected) { + t.Error(tc.tname, "Selectors not as expected", cmp.Diff(selectors, tc.expected)) + } + }) + } +} + +var _ = Describe("Helper Validation", Ordered, func() { + + var cancel context.CancelFunc + var ctx context.Context + var dc *sriovnetworkv1.SriovOperatorConfig + var in *appsv1.DaemonSet + + BeforeAll(func() { + By("Setup controller manager") + k8sManager, err := setupK8sManagerForTest() + Expect(err).ToNot(HaveOccurred()) + + ctx, cancel = context.WithCancel(context.Background()) + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + defer GinkgoRecover() + By("Start controller manager") + err := k8sManager.Start(ctx) + Expect(err).ToNot(HaveOccurred()) + }() + + DeferCleanup(func() { + By("Shutdown controller manager") + cancel() + wg.Wait() + }) + }) + + BeforeEach(func() { + dc = &sriovnetworkv1.SriovOperatorConfig{ + ObjectMeta: controllerruntime.ObjectMeta{ + Name: "default", + Namespace: vars.Namespace, + UID: "12312312"}} + in = &appsv1.DaemonSet{ + ObjectMeta: controllerruntime.ObjectMeta{ + Name: "sriov-device-plugin", + Namespace: vars.Namespace}, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "sriov-device-plugin"}}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: controllerruntime.ObjectMeta{ + Labels: map[string]string{"app": "sriov-device-plugin"}}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Image: "test:latest", + Name: "test", + }, + }, + }, + }}} + + err := k8sClient.Delete(ctx, in) + if err != nil { + Expect(errors.IsNotFound(err)).To(BeTrue()) + } + }) + + Context("syncDaemonSet", func() { + It("should create a new daemon", func() { + pl := &sriovnetworkv1.SriovNetworkNodePolicyList{Items: []sriovnetworkv1.SriovNetworkNodePolicy{ + {ObjectMeta: controllerruntime.ObjectMeta{Name: "test", Namespace: vars.Namespace}}, + }} + err := syncDaemonSet(ctx, k8sClient, vars.Scheme, dc, pl, in) + Expect(err).ToNot(HaveOccurred()) + Expect(in.Spec.Template.Spec.Affinity).To(BeNil()) + }) + It("should update affinity", func() { + pl := &sriovnetworkv1.SriovNetworkNodePolicyList{Items: []sriovnetworkv1.SriovNetworkNodePolicy{ + { + ObjectMeta: controllerruntime.ObjectMeta{ + Name: "test", + Namespace: vars.Namespace, + }, + Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ + NodeSelector: map[string]string{"test": "test"}, + }, + }, + }} + + err := k8sClient.Create(ctx, in) + Expect(err).ToNot(HaveOccurred()) + + err = syncDaemonSet(ctx, k8sClient, vars.Scheme, dc, pl, in) + Expect(err).ToNot(HaveOccurred()) + Expect(in.Spec.Template.Spec.Affinity).ToNot(BeNil()) + Expect(in.Spec.Template.Spec.Affinity.NodeAffinity).ToNot(BeNil()) + Expect(in.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution).ToNot(BeNil()) + Expect(len(in.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms)).To(Equal(1)) + }) + It("should update affinity with multiple", func() { + pl := &sriovnetworkv1.SriovNetworkNodePolicyList{Items: []sriovnetworkv1.SriovNetworkNodePolicy{ + { + ObjectMeta: controllerruntime.ObjectMeta{ + Name: "test", + Namespace: vars.Namespace, + }, + Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ + NodeSelector: map[string]string{"test": "test"}, + }, + }, + { + ObjectMeta: controllerruntime.ObjectMeta{ + Name: "test1", + Namespace: vars.Namespace, + }, + Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ + NodeSelector: map[string]string{"test1": "test"}, + }, + }, + }} + + err := k8sClient.Create(ctx, in) + Expect(err).ToNot(HaveOccurred()) + + err = syncDaemonSet(ctx, k8sClient, vars.Scheme, dc, pl, in) + Expect(err).ToNot(HaveOccurred()) + Expect(in.Spec.Template.Spec.Affinity).ToNot(BeNil()) + Expect(in.Spec.Template.Spec.Affinity.NodeAffinity).ToNot(BeNil()) + Expect(in.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution).ToNot(BeNil()) + Expect(len(in.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms)).To(Equal(2)) + }) + It("should switch affinity", func() { + pl := &sriovnetworkv1.SriovNetworkNodePolicyList{Items: []sriovnetworkv1.SriovNetworkNodePolicy{ + { + ObjectMeta: controllerruntime.ObjectMeta{ + Name: "test1", + Namespace: vars.Namespace, + }, + Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ + NodeSelector: map[string]string{"test1": "test"}, + }, + }, + }} + + in.Spec.Template.Spec.Affinity = &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{ + MatchExpressions: []corev1.NodeSelectorRequirement{{ + Operator: corev1.NodeSelectorOpIn, + Key: "test", + Values: []string{"test"}, + }}, + }}, + }, + }, + } + + err := k8sClient.Create(ctx, in) + Expect(err).ToNot(HaveOccurred()) + + err = syncDaemonSet(ctx, k8sClient, vars.Scheme, dc, pl, in) + Expect(err).ToNot(HaveOccurred()) + Expect(in.Spec.Template.Spec.Affinity).ToNot(BeNil()) + Expect(in.Spec.Template.Spec.Affinity.NodeAffinity).ToNot(BeNil()) + Expect(in.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution).ToNot(BeNil()) + Expect(len(in.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms)).To(Equal(1)) + Expect(len(in.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions)).To(Equal(1)) + Expect(in.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Key).To(Equal("test1")) + }) + }) +}) diff --git a/controllers/sriovnetworknodepolicy_controller.go b/controllers/sriovnetworknodepolicy_controller.go index c4c1fc42d..176b27076 100644 --- a/controllers/sriovnetworknodepolicy_controller.go +++ b/controllers/sriovnetworknodepolicy_controller.go @@ -25,13 +25,10 @@ import ( "strings" "time" - errs "github.com/pkg/errors" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - uns "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/workqueue" @@ -49,7 +46,6 @@ import ( sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" constants "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/featuregate" - "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/render" "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) @@ -370,66 +366,6 @@ func (r *SriovNetworkNodePolicyReconciler) syncSriovNetworkNodeState(ctx context return nil } -func setDsNodeAffinity(pl *sriovnetworkv1.SriovNetworkNodePolicyList, ds *appsv1.DaemonSet) error { - terms := nodeSelectorTermsForPolicyList(pl.Items) - if len(terms) > 0 { - ds.Spec.Template.Spec.Affinity = &corev1.Affinity{ - NodeAffinity: &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ - NodeSelectorTerms: terms, - }, - }, - } - } - return nil -} - -func nodeSelectorTermsForPolicyList(policies []sriovnetworkv1.SriovNetworkNodePolicy) []corev1.NodeSelectorTerm { - terms := []corev1.NodeSelectorTerm{} - for _, p := range policies { - // Note(adrianc): default policy is deprecated and ignored. - if p.Name == constants.DefaultPolicyName { - continue - } - - if len(p.Spec.NodeSelector) == 0 { - continue - } - expressions := []corev1.NodeSelectorRequirement{} - for k, v := range p.Spec.NodeSelector { - exp := corev1.NodeSelectorRequirement{ - Operator: corev1.NodeSelectorOpIn, - Key: k, - Values: []string{v}, - } - expressions = append(expressions, exp) - } - // sorting is needed to keep the daemon spec stable. - // the items are popped in a random order from the map - sort.Slice(expressions, func(i, j int) bool { - return expressions[i].Key < expressions[j].Key - }) - nodeSelector := corev1.NodeSelectorTerm{ - MatchExpressions: expressions, - } - terms = append(terms, nodeSelector) - } - - return terms -} - -// renderDsForCR returns a busybox pod with the same name/namespace as the cr -func renderDsForCR(path string, data *render.RenderData) ([]*uns.Unstructured, error) { - logger := log.Log.WithName("renderDsForCR") - logger.V(1).Info("Start to render objects") - - objs, err := render.RenderDir(path, data) - if err != nil { - return nil, errs.Wrap(err, "failed to render SR-IOV Network Operator manifests") - } - return objs, nil -} - func (r *SriovNetworkNodePolicyReconciler) renderDevicePluginConfigData(ctx context.Context, pl *sriovnetworkv1.SriovNetworkNodePolicyList, node *corev1.Node) (dptypes.ResourceConfList, error) { logger := log.Log.WithName("renderDevicePluginConfigData") logger.V(1).Info("Start to render device plugin config data", "node", node.Name) diff --git a/controllers/sriovnetworknodepolicy_controller_test.go b/controllers/sriovnetworknodepolicy_controller_test.go index e3d717fdd..a116efe87 100644 --- a/controllers/sriovnetworknodepolicy_controller_test.go +++ b/controllers/sriovnetworknodepolicy_controller_test.go @@ -22,120 +22,6 @@ import ( "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars" ) -func TestNodeSelectorMerge(t *testing.T) { - table := []struct { - tname string - policies []sriovnetworkv1.SriovNetworkNodePolicy - expected []corev1.NodeSelectorTerm - }{ - { - tname: "testoneselector", - policies: []sriovnetworkv1.SriovNetworkNodePolicy{ - { - Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ - NodeSelector: map[string]string{ - "foo": "bar", - }, - }, - }, - { - Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ - NodeSelector: map[string]string{ - "bb": "cc", - }, - }, - }, - }, - expected: []corev1.NodeSelectorTerm{ - { - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Operator: corev1.NodeSelectorOpIn, - Key: "foo", - Values: []string{"bar"}, - }, - }, - }, - { - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Operator: corev1.NodeSelectorOpIn, - Key: "bb", - Values: []string{"cc"}, - }, - }, - }, - }, - }, - { - tname: "testtwoselectors", - policies: []sriovnetworkv1.SriovNetworkNodePolicy{ - { - Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ - NodeSelector: map[string]string{ - "foo": "bar", - "foo1": "bar1", - }, - }, - }, - { - Spec: sriovnetworkv1.SriovNetworkNodePolicySpec{ - NodeSelector: map[string]string{ - "bb": "cc", - "bb1": "cc1", - "bb2": "cc2", - }, - }, - }, - }, - expected: []corev1.NodeSelectorTerm{ - { - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Operator: corev1.NodeSelectorOpIn, - Key: "foo", - Values: []string{"bar"}, - }, - { - Operator: corev1.NodeSelectorOpIn, - Key: "foo1", - Values: []string{"bar1"}, - }, - }, - }, - { - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Operator: corev1.NodeSelectorOpIn, - Key: "bb", - Values: []string{"cc"}, - }, - { - Operator: corev1.NodeSelectorOpIn, - Key: "bb1", - Values: []string{"cc1"}, - }, - { - Operator: corev1.NodeSelectorOpIn, - Key: "bb2", - Values: []string{"cc2"}, - }, - }, - }, - }, - }, - } - - for _, tc := range table { - t.Run(tc.tname, func(t *testing.T) { - selectors := nodeSelectorTermsForPolicyList(tc.policies) - if !cmp.Equal(selectors, tc.expected) { - t.Error(tc.tname, "Selectors not as expected", cmp.Diff(selectors, tc.expected)) - } - }) - } -} - func mustMarshallSelector(t *testing.T, input *dptypes.NetDeviceSelectors) *json.RawMessage { out, err := json.Marshal(input) if err != nil { From 1841fbe3f482f544e672bb8cca4b6450c75530b7 Mon Sep 17 00:00:00 2001 From: Yury Kulazhenkov Date: Tue, 9 Apr 2024 14:38:48 +0300 Subject: [PATCH 06/21] Change system dependencies for "post" systemd service This is required to be compatible with OVS installations, which use SysV init scripts (wrapped with systemd-sysv-generator). On such systems, OVS may depend on the network-online.target, so the start order of services changes (OVS will start after the "post" service). The proposed solution is to add the following explicit dependencies After: openvswitch-switch.service Before: kubelet.service. Note: the dependency is ignored if the openvswitch-switch.service is not found. Signed-off-by: Yury Kulazhenkov --- .../kubernetes/sriov-config-post-network-service.yaml | 6 +++--- .../openshift/sriov-config-service.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bindata/manifests/sriov-config-service/kubernetes/sriov-config-post-network-service.yaml b/bindata/manifests/sriov-config-service/kubernetes/sriov-config-post-network-service.yaml index ecfb4c108..08c8bce76 100644 --- a/bindata/manifests/sriov-config-service/kubernetes/sriov-config-post-network-service.yaml +++ b/bindata/manifests/sriov-config-service/kubernetes/sriov-config-post-network-service.yaml @@ -1,8 +1,8 @@ contents: | [Unit] Description=Configures SRIOV NIC - post network configuration - After=systemd-networkd-wait-online.service NetworkManager-wait-online.service - Before=network-online.target + After=systemd-networkd-wait-online.service NetworkManager-wait-online.service openvswitch-switch.service + Before=kubelet.service [Service] Type=oneshot @@ -10,6 +10,6 @@ contents: | StandardOutput=journal+console [Install] - WantedBy=network-online.target + WantedBy=multi-user.target enabled: true name: sriov-config-post-network.service diff --git a/bindata/manifests/sriov-config-service/openshift/sriov-config-service.yaml b/bindata/manifests/sriov-config-service/openshift/sriov-config-service.yaml index 62b919f6f..706679430 100644 --- a/bindata/manifests/sriov-config-service/openshift/sriov-config-service.yaml +++ b/bindata/manifests/sriov-config-service/openshift/sriov-config-service.yaml @@ -33,8 +33,8 @@ spec: # Removal of this file signals firstboot completion ConditionPathExists=!/etc/ignition-machine-config-encapsulated.json Description=Configures SRIOV NIC - post network configuration - After=systemd-networkd-wait-online.service NetworkManager-wait-online.service - Before=network-online.target + After=systemd-networkd-wait-online.service NetworkManager-wait-online.service openvswitch-switch.service + Before=kubelet.service [Service] Type=oneshot @@ -42,6 +42,6 @@ spec: StandardOutput=journal+console [Install] - WantedBy=network-online.target + WantedBy=multi-user.target enabled: true name: "sriov-config-post-network.service" From 359282a87662e50c8d27daf59927dd74348bf883 Mon Sep 17 00:00:00 2001 From: Yury Kulazhenkov Date: Fri, 12 Apr 2024 16:00:13 +0300 Subject: [PATCH 07/21] Use `systemctl reenable` command to enable services We use reenable command (the command is a combination of disable+enable) to reset symlinks for the unit and make sure that only symlinks that are currently configured in the [Install] section exist for the service. Signed-off-by: Yury Kulazhenkov --- pkg/host/internal/service/service.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/host/internal/service/service.go b/pkg/host/internal/service/service.go index f1574526f..c8705068a 100644 --- a/pkg/host/internal/service/service.go +++ b/pkg/host/internal/service/service.go @@ -97,8 +97,11 @@ func (s *service) EnableService(service *types.Service) error { } defer exit() - // Enable service - _, _, err = s.utilsHelper.RunCommand("systemctl", "enable", service.Name) + // Enable the service + // we use reenable command (the command is a combination of disable+enable) to reset + // symlinks for the unit and make sure that only symlinks that are currently + // configured in the [Install] section exist for the service. + _, _, err = s.utilsHelper.RunCommand("systemctl", "reenable", service.Name) return err } From ace40a0f6b8d32c34a05fc680130c1a358d90fbd Mon Sep 17 00:00:00 2001 From: Sebastian Sch Date: Sun, 24 Mar 2024 15:57:48 +0200 Subject: [PATCH 08/21] bump CNI version for the sriov-cni Signed-off-by: Sebastian Sch --- bindata/manifests/cni-config/sriov-cni-config.yaml | 2 +- controllers/sriovibnetwork_controller_test.go | 2 +- controllers/sriovnetwork_controller_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bindata/manifests/cni-config/sriov-cni-config.yaml b/bindata/manifests/cni-config/sriov-cni-config.yaml index 6e917e528..749e326c7 100644 --- a/bindata/manifests/cni-config/sriov-cni-config.yaml +++ b/bindata/manifests/cni-config/sriov-cni-config.yaml @@ -7,7 +7,7 @@ metadata: k8s.v1.cni.cncf.io/resourceName: {{.SriovCniResourceName}} spec: config: '{ - "cniVersion":"0.3.1", + "cniVersion":"1.0.0", "name":"{{.SriovNetworkName}}", {{- if .MetaPluginsConfigured -}} "plugins": [ diff --git a/controllers/sriovibnetwork_controller_test.go b/controllers/sriovibnetwork_controller_test.go index 9b2aef84e..badb11d97 100644 --- a/controllers/sriovibnetwork_controller_test.go +++ b/controllers/sriovibnetwork_controller_test.go @@ -285,7 +285,7 @@ func generateExpectedIBNetConfig(cr *sriovnetworkv1.SriovIBNetwork) string { if cr.Spec.IPAM != "" { ipam = cr.Spec.IPAM } - configStr, err := formatJSON(fmt.Sprintf(`{ "cniVersion":"0.3.1", "name":"%s","type":"ib-sriov",%s"ipam":%s }`, cr.GetName(), state, ipam)) + configStr, err := formatJSON(fmt.Sprintf(`{ "cniVersion":"1.0.0", "name":"%s","type":"ib-sriov",%s"ipam":%s }`, cr.GetName(), state, ipam)) if err != nil { panic(err) } diff --git a/controllers/sriovnetwork_controller_test.go b/controllers/sriovnetwork_controller_test.go index 356b7d3cd..2eceb2133 100644 --- a/controllers/sriovnetwork_controller_test.go +++ b/controllers/sriovnetwork_controller_test.go @@ -355,7 +355,7 @@ func generateExpectedNetConfig(cr *sriovnetworkv1.SriovNetwork) string { } configStr, err := formatJSON(fmt.Sprintf( - `{ "cniVersion":"0.3.1", "name":"%s","type":"sriov","vlan":%d,%s%s"vlanQoS":%d,%s%s%s%s"ipam":%s }`, + `{ "cniVersion":"1.0.0", "name":"%s","type":"sriov","vlan":%d,%s%s"vlanQoS":%d,%s%s%s%s"ipam":%s }`, cr.GetName(), cr.Spec.Vlan, spoofchk, trust, vlanQoS, vlanProto, state, logLevel, logFile, ipam)) if err != nil { panic(err) From e3ac6368d770d1876914cfdc0470ad83dfa683db Mon Sep 17 00:00:00 2001 From: Yury Kulazhenkov Date: Thu, 4 Apr 2024 15:04:37 +0300 Subject: [PATCH 09/21] Allow switchdev+externallyManged configs in webhook Current implementation supports this combination. NIC should be already in switchdev mode and have enough VFs to pass the check. Signed-off-by: Yury Kulazhenkov --- pkg/webhook/validate.go | 17 ---- pkg/webhook/validate_test.go | 157 +++++++++++++++++------------------ 2 files changed, 77 insertions(+), 97 deletions(-) diff --git a/pkg/webhook/validate.go b/pkg/webhook/validate.go index a4cca97c0..6937aecc5 100644 --- a/pkg/webhook/validate.go +++ b/pkg/webhook/validate.go @@ -225,13 +225,6 @@ func staticValidateSriovNetworkNodePolicy(cr *sriovnetworkv1.SriovNetworkNodePol if (cr.Spec.VdpaType == consts.VdpaTypeVirtio || cr.Spec.VdpaType == consts.VdpaTypeVhost) && cr.Spec.EswitchMode != sriovnetworkv1.ESwithModeSwitchDev { return false, fmt.Errorf("vdpa requires the device to be configured in switchdev mode") } - - // Externally created: we don't support ExternallyManaged + EswitchMode - //TODO: if needed we will need to add this in the future as today EswitchMode is for HWOFFLOAD - if cr.Spec.ExternallyManaged && cr.Spec.EswitchMode == sriovnetworkv1.ESwithModeSwitchDev { - return false, fmt.Errorf("ExternallyManaged doesn't support the device to be configured in switchdev mode") - } - return true, nil } @@ -426,16 +419,6 @@ func validateExternallyManage(current, previous *sriovnetworkv1.SriovNetworkNode return fmt.Errorf("externallyManage is inconsistent with existing policy %s", previous.GetName()) } - // reject policy with externallyManage if there is a policy on the same PF with switch dev - if current.Spec.ExternallyManaged && previous.Spec.EswitchMode == sriovnetworkv1.ESwithModeSwitchDev { - return fmt.Errorf("externallyManage overlap with switchdev mode in existing policy %s", previous.GetName()) - } - - // reject policy with externallyManage if there is a policy on the same PF with switch dev - if previous.Spec.ExternallyManaged && current.Spec.EswitchMode == sriovnetworkv1.ESwithModeSwitchDev { - return fmt.Errorf("switchdev overlap with externallyManage mode in existing policy %s", previous.GetName()) - } - return nil } diff --git a/pkg/webhook/validate_test.go b/pkg/webhook/validate_test.go index ccb69a5b7..259aedbce 100644 --- a/pkg/webhook/validate_test.go +++ b/pkg/webhook/validate_test.go @@ -472,62 +472,6 @@ func TestValidatePolicyForNodePolicyWithExternallyManageConflict(t *testing.T) { g.Expect(err).To(MatchError(ContainSubstring(fmt.Sprintf("externallyManage is inconsistent with existing policy %s", appliedPolicy.ObjectMeta.Name)))) } -func TestValidatePolicyForNodePolicyWithExternallyManageConflictWithSwitchDev(t *testing.T) { - appliedPolicy := newNodePolicy() - appliedPolicy.Spec.EswitchMode = ESwithModeSwitchDev - - policy := &SriovNetworkNodePolicy{ - ObjectMeta: metav1.ObjectMeta{ - Name: "p0", - }, - Spec: SriovNetworkNodePolicySpec{ - DeviceType: "netdevice", - NicSelector: SriovNetworkNicSelector{ - PfNames: []string{"ens803f1#3-4"}, - Vendor: "8086", - }, - NodeSelector: map[string]string{ - "feature.node.kubernetes.io/network-sriov.capable": "true", - }, - NumVfs: 63, - Priority: 99, - ResourceName: "p0", - ExternallyManaged: true, - }, - } - g := NewGomegaWithT(t) - err := validatePolicyForNodePolicy(policy, appliedPolicy) - g.Expect(err).To(HaveOccurred()) -} - -func TestValidatePolicyForNodePolicyWithSwitchDevConflictWithExternallyManage(t *testing.T) { - appliedPolicy := newNodePolicy() - appliedPolicy.Spec.ExternallyManaged = true - - policy := &SriovNetworkNodePolicy{ - ObjectMeta: metav1.ObjectMeta{ - Name: "p0", - }, - Spec: SriovNetworkNodePolicySpec{ - DeviceType: "netdevice", - NicSelector: SriovNetworkNicSelector{ - PfNames: []string{"ens803f1#3-4"}, - Vendor: "8086", - }, - NodeSelector: map[string]string{ - "feature.node.kubernetes.io/network-sriov.capable": "true", - }, - NumVfs: 63, - Priority: 99, - ResourceName: "p0", - EswitchMode: ESwithModeSwitchDev, - }, - } - g := NewGomegaWithT(t) - err := validatePolicyForNodePolicy(policy, appliedPolicy) - g.Expect(err).To(HaveOccurred()) -} - func TestValidatePolicyForNodeStateWithExternallyManageAndMTU(t *testing.T) { state := newNodeState() policy := &SriovNetworkNodePolicy{ @@ -1024,30 +968,6 @@ func TestStaticValidateSriovNetworkNodePolicyVhostVdpaMustSpecifySwitchDev(t *te g.Expect(ok).To(Equal(false)) } -func TestStaticValidateSriovNetworkNodePolicyWithExternallyCreatedAndSwitchDev(t *testing.T) { - policy := &SriovNetworkNodePolicy{ - Spec: SriovNetworkNodePolicySpec{ - DeviceType: "netdevice", - NicSelector: SriovNetworkNicSelector{ - Vendor: "8086", - DeviceID: "158b", - }, - NodeSelector: map[string]string{ - "feature.node.kubernetes.io/network-sriov.capable": "true", - }, - NumVfs: 63, - Priority: 99, - ResourceName: "p0", - EswitchMode: "switchdev", - ExternallyManaged: true, - }, - } - g := NewGomegaWithT(t) - ok, err := staticValidateSriovNetworkNodePolicy(policy) - g.Expect(err).To(HaveOccurred()) - g.Expect(ok).To(BeFalse()) -} - func TestValidatePolicyForNodeStateVirtioVdpaWithNotSupportedVendor(t *testing.T) { state := newNodeState() policy := &SriovNetworkNodePolicy{ @@ -1273,3 +1193,80 @@ func TestValidatePolicyForNodeStateWithValidVFAndNetFilter(t *testing.T) { g.Expect(err).NotTo(HaveOccurred()) g.Expect(interfaceSelected).To(Equal(true)) } + +func TestValidatePolicyForNodeStateWithExternallyManageAndSwitchdev(t *testing.T) { + state := newNodeState() + policy := &SriovNetworkNodePolicy{ + Spec: SriovNetworkNodePolicySpec{ + DeviceType: "netdevice", + NicSelector: SriovNetworkNicSelector{ + PfNames: []string{"ens803f0"}, + RootDevices: []string{"0000:86:00.0"}, + Vendor: "8086", + }, + NodeSelector: map[string]string{ + "feature.node.kubernetes.io/network-sriov.capable": "true", + }, + NumVfs: 4, + ResourceName: "p0", + EswitchMode: "switchdev", + ExternallyManaged: true, + }, + } + g := NewGomegaWithT(t) + _, err := validatePolicyForNodeState(policy, state, NewNode()) + g.Expect(err).NotTo(HaveOccurred()) +} + +func TestValidatePolicyForNodeStateWithExternallyManageAndSwitchdevAndWrongVFCount(t *testing.T) { + state := newNodeState() + policy := &SriovNetworkNodePolicy{ + Spec: SriovNetworkNodePolicySpec{ + DeviceType: "netdevice", + NicSelector: SriovNetworkNicSelector{ + PfNames: []string{"ens803f0"}, + RootDevices: []string{"0000:86:00.0"}, + Vendor: "8086", + }, + NodeSelector: map[string]string{ + "feature.node.kubernetes.io/network-sriov.capable": "true", + }, + NumVfs: 30, + ResourceName: "p0", + EswitchMode: "switchdev", + ExternallyManaged: true, + }, + } + g := NewGomegaWithT(t) + _, err := validatePolicyForNodeState(policy, state, NewNode()) + g.Expect(err).To(MatchError(ContainSubstring("is higher than the virtual functions allocated for the PF externally value"))) +} + +func TestValidatePolicyForNodePolicyAllowSwitchdevWithExternallyManage(t *testing.T) { + appliedPolicy := newNodePolicy() + appliedPolicy.Spec.ExternallyManaged = true + + policy := &SriovNetworkNodePolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "p0", + }, + Spec: SriovNetworkNodePolicySpec{ + DeviceType: "netdevice", + NicSelector: SriovNetworkNicSelector{ + PfNames: []string{"ens803f1#3-4"}, + Vendor: "8086", + }, + NodeSelector: map[string]string{ + "feature.node.kubernetes.io/network-sriov.capable": "true", + }, + NumVfs: 63, + Priority: 99, + ResourceName: "p0", + ExternallyManaged: true, + EswitchMode: ESwithModeSwitchDev, + }, + } + g := NewGomegaWithT(t) + err := validatePolicyForNodePolicy(policy, appliedPolicy) + g.Expect(err).NotTo(HaveOccurred()) +} From 62a6f01ed422e5e7abf60f90d169a11dd8e6a5ab Mon Sep 17 00:00:00 2001 From: Sebastian Sch Date: Thu, 18 Apr 2024 17:35:39 +0300 Subject: [PATCH 10/21] add support to specifi cluster version for k8s CI Signed-off-by: Sebastian Sch --- hack/run-e2e-conformance-virtual-cluster.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hack/run-e2e-conformance-virtual-cluster.sh b/hack/run-e2e-conformance-virtual-cluster.sh index 2c40f1ad1..ab6d6e9af 100755 --- a/hack/run-e2e-conformance-virtual-cluster.sh +++ b/hack/run-e2e-conformance-virtual-cluster.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash set -xeo pipefail +cluster_version=${CLUSTER_VERSION:-1.29.3} cluster_name=${CLUSTER_NAME:-virtual} domain_name=$cluster_name.lab @@ -52,6 +53,7 @@ kcli create network -c 192.168.124.0/24 k8s kcli create network -c 192.168.${virtual_router_id}.0/24 --nodhcp -i $cluster_name cat < ./${cluster_name}-plan.yaml +version: $cluster_version ctlplane_memory: 4096 worker_memory: 4096 pool: default From 707a7570133f8c34efb914b02aba2bf1a03d74cc Mon Sep 17 00:00:00 2001 From: Marcelo Guerrero Date: Thu, 26 Oct 2023 10:07:24 +0200 Subject: [PATCH 11/21] Remove LastState parameter of GenericPlugin The generic plugin was applying config changes only if the desired spec of interfaces was different from the last applied spec. This logic is different from the one in OnNodeStateChange where the real status of the interfaces is used to detect changes. By removing the LastState parameter (and related code), the generic plugin will also use the real status of interfaces to decide whether to apply changes or not. The SyncNodeState function has this logic. --- pkg/plugins/generic/generic_plugin.go | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/pkg/plugins/generic/generic_plugin.go b/pkg/plugins/generic/generic_plugin.go index a7a71ca15..274f0dc01 100644 --- a/pkg/plugins/generic/generic_plugin.go +++ b/pkg/plugins/generic/generic_plugin.go @@ -4,7 +4,6 @@ import ( "bytes" "errors" "os/exec" - "reflect" "strconv" "strings" "syscall" @@ -52,7 +51,6 @@ type GenericPlugin struct { PluginName string SpecVersion string DesireState *sriovnetworkv1.SriovNetworkNodeState - LastState *sriovnetworkv1.SriovNetworkNodeState DriverStateMap DriverStateMapType DesiredKernelArgs map[string]bool helpers helper.HostHelpersInterface @@ -159,14 +157,6 @@ func (p *GenericPlugin) syncDriverState() error { func (p *GenericPlugin) Apply() error { log.Log.Info("generic plugin Apply()", "desiredState", p.DesireState.Spec) - if p.LastState != nil { - log.Log.Info("generic plugin Apply()", "lastState", p.LastState.Spec) - if reflect.DeepEqual(p.LastState.Spec.Interfaces, p.DesireState.Spec.Interfaces) { - log.Log.Info("generic plugin Apply(): desired and latest state are the same, nothing to apply") - return nil - } - } - if err := p.syncDriverState(); err != nil { return err } @@ -188,8 +178,7 @@ func (p *GenericPlugin) Apply() error { } return err } - p.LastState = &sriovnetworkv1.SriovNetworkNodeState{} - *p.LastState = *p.DesireState + return nil } From 82af069fb5d9609eae66fdbd07e2602412f1adbc Mon Sep 17 00:00:00 2001 From: Marcelo Guerrero Date: Mon, 30 Oct 2023 16:58:16 +0100 Subject: [PATCH 12/21] Verify status changes on managed interfaces Users could modify the settings of VFs which have been configured by the sriov operator. This PR starts the reconciliation loop when these changes are detected in the generic plugin. Signed-off-by: Marcelo Guerrero --- pkg/daemon/daemon.go | 164 ++++++++++++--------- pkg/plugins/fake/fake_plugin.go | 4 + pkg/plugins/generic/generic_plugin.go | 25 ++++ pkg/plugins/generic/generic_plugin_test.go | 100 +++++++++++++ pkg/plugins/intel/intel_plugin.go | 9 +- pkg/plugins/k8s/k8s_plugin.go | 6 + pkg/plugins/mellanox/mellanox_plugin.go | 6 + pkg/plugins/mock/mock_plugin.go | 22 ++- pkg/plugins/plugin.go | 10 +- pkg/plugins/virtual/virtual_plugin.go | 5 + 10 files changed, 272 insertions(+), 79 deletions(-) diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index 07eefb4f8..65964d7bc 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -313,77 +313,60 @@ func (dn *Daemon) nodeStateSyncHandler() error { latest := dn.desiredNodeState.GetGeneration() log.Log.V(0).Info("nodeStateSyncHandler(): new generation", "generation", latest) - if dn.currentNodeState.GetGeneration() == latest && !dn.isDrainCompleted() { - if vars.UsingSystemdMode { - serviceEnabled, err := dn.HostHelpers.IsServiceEnabled(systemd.SriovServicePath) - if err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): failed to check if sriov-config service exist on host") - return err - } - postNetworkServiceEnabled, err := dn.HostHelpers.IsServiceEnabled(systemd.SriovPostNetworkServicePath) - if err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): failed to check if sriov-config-post-network service exist on host") - return err - } - - // if the service doesn't exist we should continue to let the k8s plugin to create the service files - // this is only for k8s base environments, for openshift the sriov-operator creates a machine config to will apply - // the system service and reboot the node the config-daemon doesn't need to do anything. - if !(serviceEnabled && postNetworkServiceEnabled) { - sriovResult = &systemd.SriovResult{SyncStatus: consts.SyncStatusFailed, - LastSyncError: fmt.Sprintf("some sriov systemd services are not available on node: "+ - "sriov-config available:%t, sriov-config-post-network available:%t", serviceEnabled, postNetworkServiceEnabled)} - } else { - sriovResult, err = systemd.ReadSriovResult() - if err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): failed to load sriov result file from host") - return err - } - } - if sriovResult.LastSyncError != "" || sriovResult.SyncStatus == consts.SyncStatusFailed { - log.Log.Info("nodeStateSyncHandler(): sync failed systemd service error", "last-sync-error", sriovResult.LastSyncError) - - // add the error but don't requeue - dn.refreshCh <- Message{ - syncStatus: consts.SyncStatusFailed, - lastSyncError: sriovResult.LastSyncError, - } - <-dn.syncCh - return nil - } - } - log.Log.V(0).Info("nodeStateSyncHandler(): Interface not changed") - if dn.desiredNodeState.Status.LastSyncError != "" || - dn.desiredNodeState.Status.SyncStatus != consts.SyncStatusSucceeded { - dn.refreshCh <- Message{ - syncStatus: consts.SyncStatusSucceeded, - lastSyncError: "", - } - // wait for writer to refresh the status - <-dn.syncCh + // load plugins if it has not loaded + if len(dn.loadedPlugins) == 0 { + dn.loadedPlugins, err = loadPlugins(dn.desiredNodeState, dn.HostHelpers, dn.disabledPlugins) + if err != nil { + log.Log.Error(err, "nodeStateSyncHandler(): failed to enable vendor plugins") + return err } - - return nil } - if dn.desiredNodeState.GetGeneration() == 1 && len(dn.desiredNodeState.Spec.Interfaces) == 0 { - err = dn.HostHelpers.ClearPCIAddressFolder() + if vars.UsingSystemdMode && dn.currentNodeState.GetGeneration() == latest && !dn.isDrainCompleted() { + serviceEnabled, err := dn.HostHelpers.IsServiceEnabled(systemd.SriovServicePath) if err != nil { - log.Log.Error(err, "failed to clear the PCI address configuration") + log.Log.Error(err, "nodeStateSyncHandler(): failed to check if sriov-config service exist on host") + return err + } + postNetworkServiceEnabled, err := dn.HostHelpers.IsServiceEnabled(systemd.SriovPostNetworkServicePath) + if err != nil { + log.Log.Error(err, "nodeStateSyncHandler(): failed to check if sriov-config-post-network service exist on host") return err } - log.Log.V(0).Info( - "nodeStateSyncHandler(): interface policy spec not yet set by controller for sriovNetworkNodeState", - "name", dn.desiredNodeState.Name) - if dn.desiredNodeState.Status.SyncStatus != "Succeeded" { + // if the service doesn't exist we should continue to let the k8s plugin to create the service files + // this is only for k8s base environments, for openshift the sriov-operator creates a machine config to will apply + // the system service and reboot the node the config-daemon doesn't need to do anything. + if !(serviceEnabled && postNetworkServiceEnabled) { + sriovResult = &systemd.SriovResult{SyncStatus: consts.SyncStatusFailed, + LastSyncError: fmt.Sprintf("some sriov systemd services are not available on node: "+ + "sriov-config available:%t, sriov-config-post-network available:%t", serviceEnabled, postNetworkServiceEnabled)} + } else { + sriovResult, err = systemd.ReadSriovResult() + if err != nil { + log.Log.Error(err, "nodeStateSyncHandler(): failed to load sriov result file from host") + return err + } + } + if sriovResult.LastSyncError != "" || sriovResult.SyncStatus == consts.SyncStatusFailed { + log.Log.Info("nodeStateSyncHandler(): sync failed systemd service error", "last-sync-error", sriovResult.LastSyncError) + + // add the error but don't requeue dn.refreshCh <- Message{ - syncStatus: "Succeeded", - lastSyncError: "", + syncStatus: consts.SyncStatusFailed, + lastSyncError: sriovResult.LastSyncError, } - // wait for writer to refresh status <-dn.syncCh + return nil } + } + + skip, err := dn.shouldSkipReconciliation(dn.desiredNodeState) + if err != nil { + return err + } + // Reconcile only when there are changes in the spec or status of managed VFs. + if skip { return nil } @@ -404,15 +387,6 @@ func (dn *Daemon) nodeStateSyncHandler() error { } dn.desiredNodeState.Status = updatedState.Status - // load plugins if it has not loaded - if len(dn.loadedPlugins) == 0 { - dn.loadedPlugins, err = loadPlugins(dn.desiredNodeState, dn.HostHelpers, dn.disabledPlugins) - if err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): failed to enable vendor plugins") - return err - } - } - reqReboot := false reqDrain := false @@ -564,6 +538,58 @@ func (dn *Daemon) nodeStateSyncHandler() error { return nil } +func (dn *Daemon) shouldSkipReconciliation(latestState *sriovnetworkv1.SriovNetworkNodeState) (bool, error) { + var err error + + // Skip when SriovNetworkNodeState object has just been created. + if latestState.GetGeneration() == 1 && len(latestState.Spec.Interfaces) == 0 { + err = dn.HostHelpers.ClearPCIAddressFolder() + if err != nil { + log.Log.Error(err, "failed to clear the PCI address configuration") + return false, err + } + + log.Log.V(0).Info( + "shouldSkipReconciliation(): interface policy spec not yet set by controller for sriovNetworkNodeState", + "name", latestState.Name) + if latestState.Status.SyncStatus != consts.SyncStatusSucceeded { + dn.refreshCh <- Message{ + syncStatus: consts.SyncStatusSucceeded, + lastSyncError: "", + } + // wait for writer to refresh status + <-dn.syncCh + } + return true, nil + } + + // Verify changes in the spec of the SriovNetworkNodeState CR. + if dn.currentNodeState.GetGeneration() == latestState.GetGeneration() && !dn.isDrainCompleted() { + genericPlugin, ok := dn.loadedPlugins[GenericPluginName] + if ok { + // Verify changes in the status of the SriovNetworkNodeState CR. + if genericPlugin.CheckStatusChanges(latestState) { + return false, nil + } + } + + log.Log.V(0).Info("shouldSkipReconciliation(): Interface not changed") + if latestState.Status.LastSyncError != "" || + latestState.Status.SyncStatus != consts.SyncStatusSucceeded { + dn.refreshCh <- Message{ + syncStatus: consts.SyncStatusSucceeded, + lastSyncError: "", + } + // wait for writer to refresh the status + <-dn.syncCh + } + + return true, nil + } + + return false, nil +} + // handleDrain: adds the right annotation to the node and nodeState object // returns true if we need to finish the reconcile loop and wait for a new object func (dn *Daemon) handleDrain(reqReboot bool) (bool, error) { diff --git a/pkg/plugins/fake/fake_plugin.go b/pkg/plugins/fake/fake_plugin.go index f7fe6e56f..11c30d6ea 100644 --- a/pkg/plugins/fake/fake_plugin.go +++ b/pkg/plugins/fake/fake_plugin.go @@ -21,6 +21,10 @@ func (f *FakePlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeState return false, false, nil } +func (f *FakePlugin) CheckStatusChanges(new *sriovnetworkv1.SriovNetworkNodeState) bool { + return false +} + func (f *FakePlugin) Apply() error { return nil } diff --git a/pkg/plugins/generic/generic_plugin.go b/pkg/plugins/generic/generic_plugin.go index 274f0dc01..20ef4a49c 100644 --- a/pkg/plugins/generic/generic_plugin.go +++ b/pkg/plugins/generic/generic_plugin.go @@ -139,6 +139,31 @@ func (p *GenericPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeSt return } +// CheckStatusChanges verify whether SriovNetworkNodeState CR status present changes on configured VFs. +func (p *GenericPlugin) CheckStatusChanges(new *sriovnetworkv1.SriovNetworkNodeState) bool { + log.Log.Info("generic-plugin CheckStatusChanges()") + + changed := false + for _, iface := range new.Spec.Interfaces { + found := false + for _, ifaceStatus := range new.Status.Interfaces { + if iface.PciAddress == ifaceStatus.PciAddress && !iface.ExternallyManaged { + found = true + if sriovnetworkv1.NeedToUpdateSriov(&iface, &ifaceStatus) { + changed = true + log.Log.Info("CheckStatusChanges(): status changed for interface", "address", iface.PciAddress) + } + break + } + } + + if !found { + log.Log.Info("CheckStatusChanges(): no status found for interface", "address", iface.PciAddress) + } + } + return changed +} + func (p *GenericPlugin) syncDriverState() error { for _, driverState := range p.DriverStateMap { if !driverState.DriverLoaded && driverState.NeedDriverFunc(p.DesireState, driverState) { diff --git a/pkg/plugins/generic/generic_plugin_test.go b/pkg/plugins/generic/generic_plugin_test.go index 9f9c1be83..6b75535bb 100644 --- a/pkg/plugins/generic/generic_plugin_test.go +++ b/pkg/plugins/generic/generic_plugin_test.go @@ -137,6 +137,106 @@ var _ = Describe("Generic plugin", func() { Expect(needDrain).To(BeTrue()) }) + It("should not detect changes on status", func() { + networkNodeState := &sriovnetworkv1.SriovNetworkNodeState{ + Spec: sriovnetworkv1.SriovNetworkNodeStateSpec{ + Interfaces: sriovnetworkv1.Interfaces{{ + PciAddress: "0000:00:00.0", + NumVfs: 1, + VfGroups: []sriovnetworkv1.VfGroup{{ + DeviceType: "netdevice", + PolicyName: "policy-1", + ResourceName: "resource-1", + VfRange: "0-0", + }}}}, + }, + Status: sriovnetworkv1.SriovNetworkNodeStateStatus{ + Interfaces: sriovnetworkv1.InterfaceExts{{ + PciAddress: "0000:00:00.0", + NumVfs: 1, + TotalVfs: 1, + DeviceID: "1015", + Vendor: "15b3", + Name: "sriovif1", + Mtu: 1500, + Mac: "0c:42:a1:55:ee:46", + Driver: "mlx5_core", + EswitchMode: "legacy", + LinkSpeed: "25000 Mb/s", + LinkType: "ETH", + VFs: []sriovnetworkv1.VirtualFunction{{ + PciAddress: "0000:00:00.1", + DeviceID: "1016", + Vendor: "15b3", + VfID: 0, + Name: "sriovif1v0", + Mtu: 1500, + Mac: "8e:d6:2c:62:87:1b", + Driver: "mlx5_core", + }}, + }}, + }, + } + + changed := genericPlugin.CheckStatusChanges(networkNodeState) + Expect(err).ToNot(HaveOccurred()) + Expect(changed).To(BeFalse()) + }) + + It("should detect changes on status", func() { + networkNodeState := &sriovnetworkv1.SriovNetworkNodeState{ + Spec: sriovnetworkv1.SriovNetworkNodeStateSpec{ + Interfaces: sriovnetworkv1.Interfaces{{ + PciAddress: "0000:00:00.0", + NumVfs: 2, + Mtu: 1500, + VfGroups: []sriovnetworkv1.VfGroup{{ + DeviceType: "netdevice", + PolicyName: "policy-1", + ResourceName: "resource-1", + VfRange: "0-1", + Mtu: 1500, + }}}}, + }, + Status: sriovnetworkv1.SriovNetworkNodeStateStatus{ + Interfaces: sriovnetworkv1.InterfaceExts{{ + PciAddress: "0000:00:00.0", + NumVfs: 2, + TotalVfs: 2, + DeviceID: "1015", + Vendor: "15b3", + Name: "sriovif1", + Mtu: 1500, + Mac: "0c:42:a1:55:ee:46", + Driver: "mlx5_core", + EswitchMode: "legacy", + LinkSpeed: "25000 Mb/s", + LinkType: "ETH", + VFs: []sriovnetworkv1.VirtualFunction{{ + PciAddress: "0000:00:00.1", + DeviceID: "1016", + Vendor: "15b3", + VfID: 0, + Driver: "mlx5_core", + Name: "sriovif1v0", + Mtu: 999, // Bad MTU value, changed by the user application + Mac: "8e:d6:2c:62:87:1b", + }, { + PciAddress: "0000:00:00.2", + DeviceID: "1016", + Vendor: "15b3", + VfID: 0, + Driver: "mlx5_core", + }}, + }}, + }, + } + + changed := genericPlugin.CheckStatusChanges(networkNodeState) + Expect(err).ToNot(HaveOccurred()) + Expect(changed).To(BeTrue()) + }) + It("should load vfio_pci driver", func() { networkNodeState := &sriovnetworkv1.SriovNetworkNodeState{ Spec: sriovnetworkv1.SriovNetworkNodeStateSpec{ diff --git a/pkg/plugins/intel/intel_plugin.go b/pkg/plugins/intel/intel_plugin.go index 467116ddb..a9174bdba 100644 --- a/pkg/plugins/intel/intel_plugin.go +++ b/pkg/plugins/intel/intel_plugin.go @@ -35,11 +35,16 @@ func (p *IntelPlugin) Spec() string { } // OnNodeStateChange Invoked when SriovNetworkNodeState CR is created or updated, return if need dain and/or reboot node -func (p *IntelPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeState) (needDrain bool, needReboot bool, err error) { - log.Log.Info("intel plugin OnNodeStateChange()") +func (p *IntelPlugin) OnNodeStateChange(*sriovnetworkv1.SriovNetworkNodeState) (bool, bool, error) { + log.Log.Info("intel-plugin OnNodeStateChange()") return false, false, nil } +// OnNodeStatusChange verify whether SriovNetworkNodeState CR status present changes on configured VFs. +func (p *IntelPlugin) CheckStatusChanges(*sriovnetworkv1.SriovNetworkNodeState) bool { + return false +} + // Apply config change func (p *IntelPlugin) Apply() error { log.Log.Info("intel plugin Apply()") diff --git a/pkg/plugins/k8s/k8s_plugin.go b/pkg/plugins/k8s/k8s_plugin.go index 118073ee5..aa28c0aa6 100644 --- a/pkg/plugins/k8s/k8s_plugin.go +++ b/pkg/plugins/k8s/k8s_plugin.go @@ -154,6 +154,12 @@ func (p *K8sPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeState) return } +// TODO: implement - https://github.com/k8snetworkplumbingwg/sriov-network-operator/issues/630 +// OnNodeStatusChange verify whether SriovNetworkNodeState CR status present changes on configured VFs. +func (p *K8sPlugin) CheckStatusChanges(*sriovnetworkv1.SriovNetworkNodeState) bool { + return false +} + // Apply config change func (p *K8sPlugin) Apply() error { log.Log.Info("k8s plugin Apply()") diff --git a/pkg/plugins/mellanox/mellanox_plugin.go b/pkg/plugins/mellanox/mellanox_plugin.go index 6b1b943de..1aa1f6434 100644 --- a/pkg/plugins/mellanox/mellanox_plugin.go +++ b/pkg/plugins/mellanox/mellanox_plugin.go @@ -171,6 +171,12 @@ func (p *MellanoxPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeS return } +// TODO: implement - https://github.com/k8snetworkplumbingwg/sriov-network-operator/issues/631 +// OnNodeStatusChange verify whether SriovNetworkNodeState CR status present changes on configured VFs. +func (p *MellanoxPlugin) CheckStatusChanges(*sriovnetworkv1.SriovNetworkNodeState) bool { + return false +} + // Apply config change func (p *MellanoxPlugin) Apply() error { if p.helpers.IsKernelLockdownMode() { diff --git a/pkg/plugins/mock/mock_plugin.go b/pkg/plugins/mock/mock_plugin.go index b821139c5..044d30f0e 100644 --- a/pkg/plugins/mock/mock_plugin.go +++ b/pkg/plugins/mock/mock_plugin.go @@ -48,6 +48,20 @@ func (mr *MockVendorPluginMockRecorder) Apply() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Apply", reflect.TypeOf((*MockVendorPlugin)(nil).Apply)) } +// CheckStatusChanges mocks base method. +func (m *MockVendorPlugin) CheckStatusChanges(arg0 *v1.SriovNetworkNodeState) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CheckStatusChanges", arg0) + ret0, _ := ret[0].(bool) + return ret0 +} + +// CheckStatusChanges indicates an expected call of CheckStatusChanges. +func (mr *MockVendorPluginMockRecorder) CheckStatusChanges(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckStatusChanges", reflect.TypeOf((*MockVendorPlugin)(nil).CheckStatusChanges), arg0) +} + // Name mocks base method. func (m *MockVendorPlugin) Name() string { m.ctrl.T.Helper() @@ -63,9 +77,9 @@ func (mr *MockVendorPluginMockRecorder) Name() *gomock.Call { } // OnNodeStateChange mocks base method. -func (m *MockVendorPlugin) OnNodeStateChange(new *v1.SriovNetworkNodeState) (bool, bool, error) { +func (m *MockVendorPlugin) OnNodeStateChange(arg0 *v1.SriovNetworkNodeState) (bool, bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "OnNodeStateChange", new) + ret := m.ctrl.Call(m, "OnNodeStateChange", arg0) ret0, _ := ret[0].(bool) ret1, _ := ret[1].(bool) ret2, _ := ret[2].(error) @@ -73,9 +87,9 @@ func (m *MockVendorPlugin) OnNodeStateChange(new *v1.SriovNetworkNodeState) (boo } // OnNodeStateChange indicates an expected call of OnNodeStateChange. -func (mr *MockVendorPluginMockRecorder) OnNodeStateChange(new interface{}) *gomock.Call { +func (mr *MockVendorPluginMockRecorder) OnNodeStateChange(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OnNodeStateChange", reflect.TypeOf((*MockVendorPlugin)(nil).OnNodeStateChange), new) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OnNodeStateChange", reflect.TypeOf((*MockVendorPlugin)(nil).OnNodeStateChange), arg0) } // Spec mocks base method. diff --git a/pkg/plugins/plugin.go b/pkg/plugins/plugin.go index 276e37cf2..3136dfd46 100644 --- a/pkg/plugins/plugin.go +++ b/pkg/plugins/plugin.go @@ -6,12 +6,14 @@ import ( //go:generate ../../bin/mockgen -destination mock/mock_plugin.go -source plugin.go type VendorPlugin interface { - // Return the name of plugin + // Name returns the name of plugin Name() string - // Return the SpecVersion followed by plugin + // Spec returns the SpecVersion followed by plugin Spec() string - // Invoked when SriovNetworkNodeState CR is created or updated, return if need dain and/or reboot node - OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeState) (bool, bool, error) + // OnNodeStateChange is invoked when SriovNetworkNodeState CR is created or updated, return if need dain and/or reboot node + OnNodeStateChange(*sriovnetworkv1.SriovNetworkNodeState) (bool, bool, error) // Apply config change Apply() error + // CheckStatusChanges checks status changes on the SriovNetworkNodeState CR for configured VFs. + CheckStatusChanges(*sriovnetworkv1.SriovNetworkNodeState) bool } diff --git a/pkg/plugins/virtual/virtual_plugin.go b/pkg/plugins/virtual/virtual_plugin.go index a942aaa86..cd43b120b 100644 --- a/pkg/plugins/virtual/virtual_plugin.go +++ b/pkg/plugins/virtual/virtual_plugin.go @@ -67,6 +67,11 @@ func (p *VirtualPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeSt return } +// OnNodeStatusChange verify whether SriovNetworkNodeState CR status present changes on configured VFs. +func (p *VirtualPlugin) CheckStatusChanges(*sriovnetworkv1.SriovNetworkNodeState) bool { + return false +} + // Apply config change func (p *VirtualPlugin) Apply() error { log.Log.Info("virtual plugin Apply()", "desired-state", p.DesireState.Spec) From 21d919c487828759a3236e36534703c7e476d2ef Mon Sep 17 00:00:00 2001 From: Marcelo Guerrero Date: Mon, 22 Jan 2024 18:55:58 +0100 Subject: [PATCH 13/21] Abstract logic to check and load missing KArgs Logic to check missing kernel arguments is placed in a method to be used by both OnNodeStateChange and CheckStatusChanges. --- pkg/daemon/daemon.go | 9 +- pkg/plugins/fake/fake_plugin.go | 4 +- pkg/plugins/generic/generic_plugin.go | 104 +++++++++++++-------- pkg/plugins/generic/generic_plugin_test.go | 65 ++++++++++++- pkg/plugins/intel/intel_plugin.go | 4 +- pkg/plugins/k8s/k8s_plugin.go | 4 +- pkg/plugins/mellanox/mellanox_plugin.go | 4 +- pkg/plugins/mock/mock_plugin.go | 5 +- pkg/plugins/plugin.go | 2 +- pkg/plugins/virtual/virtual_plugin.go | 4 +- 10 files changed, 147 insertions(+), 58 deletions(-) diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index 65964d7bc..950287a16 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -565,10 +565,13 @@ func (dn *Daemon) shouldSkipReconciliation(latestState *sriovnetworkv1.SriovNetw // Verify changes in the spec of the SriovNetworkNodeState CR. if dn.currentNodeState.GetGeneration() == latestState.GetGeneration() && !dn.isDrainCompleted() { - genericPlugin, ok := dn.loadedPlugins[GenericPluginName] - if ok { + for _, p := range dn.loadedPlugins { // Verify changes in the status of the SriovNetworkNodeState CR. - if genericPlugin.CheckStatusChanges(latestState) { + changed, err := p.CheckStatusChanges(latestState) + if err != nil { + return false, err + } + if changed { return false, nil } } diff --git a/pkg/plugins/fake/fake_plugin.go b/pkg/plugins/fake/fake_plugin.go index 11c30d6ea..0aa218f28 100644 --- a/pkg/plugins/fake/fake_plugin.go +++ b/pkg/plugins/fake/fake_plugin.go @@ -21,8 +21,8 @@ func (f *FakePlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeState return false, false, nil } -func (f *FakePlugin) CheckStatusChanges(new *sriovnetworkv1.SriovNetworkNodeState) bool { - return false +func (f *FakePlugin) CheckStatusChanges(new *sriovnetworkv1.SriovNetworkNodeState) (bool, error) { + return false, nil } func (f *FakePlugin) Apply() error { diff --git a/pkg/plugins/generic/generic_plugin.go b/pkg/plugins/generic/generic_plugin.go index 20ef4a49c..5e0827c69 100644 --- a/pkg/plugins/generic/generic_plugin.go +++ b/pkg/plugins/generic/generic_plugin.go @@ -140,28 +140,34 @@ func (p *GenericPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeSt } // CheckStatusChanges verify whether SriovNetworkNodeState CR status present changes on configured VFs. -func (p *GenericPlugin) CheckStatusChanges(new *sriovnetworkv1.SriovNetworkNodeState) bool { +func (p *GenericPlugin) CheckStatusChanges(current *sriovnetworkv1.SriovNetworkNodeState) (bool, error) { log.Log.Info("generic-plugin CheckStatusChanges()") - changed := false - for _, iface := range new.Spec.Interfaces { + for _, iface := range current.Spec.Interfaces { found := false - for _, ifaceStatus := range new.Status.Interfaces { + for _, ifaceStatus := range current.Status.Interfaces { + // TODO: remove the check for ExternallyManaged - https://github.com/k8snetworkplumbingwg/sriov-network-operator/issues/632 if iface.PciAddress == ifaceStatus.PciAddress && !iface.ExternallyManaged { found = true if sriovnetworkv1.NeedToUpdateSriov(&iface, &ifaceStatus) { - changed = true log.Log.Info("CheckStatusChanges(): status changed for interface", "address", iface.PciAddress) + return true, nil } break } } - if !found { log.Log.Info("CheckStatusChanges(): no status found for interface", "address", iface.PciAddress) } } - return changed + + missingKernelArgs, err := p.getMissingKernelArgs() + if err != nil { + log.Log.Error(err, "generic-plugin CheckStatusChanges(): failed to verify missing kernel arguments") + return false, err + } + + return len(missingKernelArgs) != 0, nil } func (p *GenericPlugin) syncDriverState() error { @@ -266,37 +272,48 @@ func (p *GenericPlugin) addToDesiredKernelArgs(karg string) { } } -// syncDesiredKernelArgs Should be called to set all the kernel arguments. Returns bool if node update is needed. -func (p *GenericPlugin) syncDesiredKernelArgs() (bool, error) { - needReboot := false +// getMissingKernelArgs gets Kernel arguments that have not been set. +func (p *GenericPlugin) getMissingKernelArgs() ([]string, error) { + missingArgs := make([]string, 0, len(p.DesiredKernelArgs)) if len(p.DesiredKernelArgs) == 0 { - return false, nil + return nil, nil } + kargs, err := p.helpers.GetCurrentKernelArgs() if err != nil { - return false, err + return nil, err } - for desiredKarg, attempted := range p.DesiredKernelArgs { - set := p.helpers.IsKernelArgsSet(kargs, desiredKarg) - if !set { - if attempted { - log.Log.V(2).Info("generic plugin syncDesiredKernelArgs(): previously attempted to set kernel arg", - "karg", desiredKarg) - } - // There is a case when we try to set the kernel argument here, the daemon could decide to not reboot because - // the daemon encountered a potentially one-time error. However we always want to make sure that the kernel - // argument is set once the daemon goes through node state sync again. - update, err := setKernelArg(desiredKarg) - if err != nil { - log.Log.Error(err, "generic plugin syncDesiredKernelArgs(): fail to set kernel arg", "karg", desiredKarg) - return false, err - } - if update { - needReboot = true - log.Log.V(2).Info("generic plugin syncDesiredKernelArgs(): need reboot for setting kernel arg", "karg", desiredKarg) - } - p.DesiredKernelArgs[desiredKarg] = true + + for desiredKarg := range p.DesiredKernelArgs { + if !p.helpers.IsKernelArgsSet(kargs, desiredKarg) { + missingArgs = append(missingArgs, desiredKarg) + } + } + return missingArgs, nil +} + +// syncDesiredKernelArgs should be called to set all the kernel arguments. Returns bool if node update is needed. +func (p *GenericPlugin) syncDesiredKernelArgs(kargs []string) (bool, error) { + needReboot := false + + for _, karg := range kargs { + if p.DesiredKernelArgs[karg] { + log.Log.V(2).Info("generic-plugin syncDesiredKernelArgs(): previously attempted to set kernel arg", + "karg", karg) + } + // There is a case when we try to set the kernel argument here, the daemon could decide to not reboot because + // the daemon encountered a potentially one-time error. However we always want to make sure that the kernel + // argument is set once the daemon goes through node state sync again. + update, err := setKernelArg(karg) + if err != nil { + log.Log.Error(err, "generic-plugin syncDesiredKernelArgs(): fail to set kernel arg", "karg", karg) + return false, err + } + if update { + needReboot = true + log.Log.V(2).Info("generic-plugin syncDesiredKernelArgs(): need reboot for setting kernel arg", "karg", karg) } + p.DesiredKernelArgs[karg] = true } return needReboot, nil } @@ -365,19 +382,28 @@ func (p *GenericPlugin) addVfioDesiredKernelArg(state *sriovnetworkv1.SriovNetwo } } -func (p *GenericPlugin) needRebootNode(state *sriovnetworkv1.SriovNetworkNodeState) (needReboot bool, err error) { - needReboot = false +func (p *GenericPlugin) needRebootNode(state *sriovnetworkv1.SriovNetworkNodeState) (bool, error) { + needReboot := false + p.addVfioDesiredKernelArg(state) - updateNode, err := p.syncDesiredKernelArgs() + missingKernelArgs, err := p.getMissingKernelArgs() if err != nil { - log.Log.Error(err, "generic plugin needRebootNode(): failed to set the desired kernel arguments") + log.Log.Error(err, "generic-plugin needRebootNode(): failed to verify missing kernel arguments") return false, err } - if updateNode { - log.Log.V(2).Info("generic plugin needRebootNode(): need reboot for updating kernel arguments") - needReboot = true + + if len(missingKernelArgs) != 0 { + needReboot, err = p.syncDesiredKernelArgs(missingKernelArgs) + if err != nil { + log.Log.Error(err, "generic-plugin needRebootNode(): failed to set the desired kernel arguments") + return false, err + } + if needReboot { + log.Log.V(2).Info("generic-plugin needRebootNode(): need reboot for updating kernel arguments") + } } + return needReboot, nil } diff --git a/pkg/plugins/generic/generic_plugin_test.go b/pkg/plugins/generic/generic_plugin_test.go index 6b75535bb..d2e9c1bbe 100644 --- a/pkg/plugins/generic/generic_plugin_test.go +++ b/pkg/plugins/generic/generic_plugin_test.go @@ -8,6 +8,7 @@ import ( . "github.com/onsi/gomega" sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1" + "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/consts" mock_helper "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/helper/mock" plugin "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/plugins" ) @@ -178,12 +179,12 @@ var _ = Describe("Generic plugin", func() { }, } - changed := genericPlugin.CheckStatusChanges(networkNodeState) + changed, err := genericPlugin.CheckStatusChanges(networkNodeState) Expect(err).ToNot(HaveOccurred()) Expect(changed).To(BeFalse()) }) - It("should detect changes on status", func() { + It("should detect changes on status due to spec mismatch", func() { networkNodeState := &sriovnetworkv1.SriovNetworkNodeState{ Spec: sriovnetworkv1.SriovNetworkNodeStateSpec{ Interfaces: sriovnetworkv1.Interfaces{{ @@ -232,7 +233,65 @@ var _ = Describe("Generic plugin", func() { }, } - changed := genericPlugin.CheckStatusChanges(networkNodeState) + changed, err := genericPlugin.CheckStatusChanges(networkNodeState) + Expect(err).ToNot(HaveOccurred()) + Expect(changed).To(BeTrue()) + }) + + It("should detect changes on status due to missing kernel args", func() { + networkNodeState := &sriovnetworkv1.SriovNetworkNodeState{ + Spec: sriovnetworkv1.SriovNetworkNodeStateSpec{ + Interfaces: sriovnetworkv1.Interfaces{{ + PciAddress: "0000:00:00.0", + NumVfs: 2, + Mtu: 1500, + VfGroups: []sriovnetworkv1.VfGroup{{ + DeviceType: "vfio-pci", + PolicyName: "policy-1", + ResourceName: "resource-1", + VfRange: "0-1", + Mtu: 1500, + }}}}, + }, + Status: sriovnetworkv1.SriovNetworkNodeStateStatus{ + Interfaces: sriovnetworkv1.InterfaceExts{{ + PciAddress: "0000:00:00.0", + NumVfs: 2, + TotalVfs: 2, + DeviceID: "159b", + Vendor: "8086", + Name: "sriovif1", + Mtu: 1500, + Mac: "0c:42:a1:55:ee:46", + Driver: "ice", + EswitchMode: "legacy", + LinkSpeed: "25000 Mb/s", + LinkType: "ETH", + VFs: []sriovnetworkv1.VirtualFunction{{ + PciAddress: "0000:00:00.1", + DeviceID: "1889", + Vendor: "8086", + VfID: 0, + Driver: "vfio-pci", + }, { + PciAddress: "0000:00:00.2", + DeviceID: "1889", + Vendor: "8086", + VfID: 1, + Driver: "vfio-pci", + }}, + }}, + }, + } + + // Load required kernel args. + genericPlugin.(*GenericPlugin).addVfioDesiredKernelArg(networkNodeState) + + hostHelper.EXPECT().GetCurrentKernelArgs().Return("", nil) + hostHelper.EXPECT().IsKernelArgsSet("", consts.KernelArgIntelIommu).Return(false) + hostHelper.EXPECT().IsKernelArgsSet("", consts.KernelArgIommuPt).Return(false) + + changed, err := genericPlugin.CheckStatusChanges(networkNodeState) Expect(err).ToNot(HaveOccurred()) Expect(changed).To(BeTrue()) }) diff --git a/pkg/plugins/intel/intel_plugin.go b/pkg/plugins/intel/intel_plugin.go index a9174bdba..e88a186c9 100644 --- a/pkg/plugins/intel/intel_plugin.go +++ b/pkg/plugins/intel/intel_plugin.go @@ -41,8 +41,8 @@ func (p *IntelPlugin) OnNodeStateChange(*sriovnetworkv1.SriovNetworkNodeState) ( } // OnNodeStatusChange verify whether SriovNetworkNodeState CR status present changes on configured VFs. -func (p *IntelPlugin) CheckStatusChanges(*sriovnetworkv1.SriovNetworkNodeState) bool { - return false +func (p *IntelPlugin) CheckStatusChanges(*sriovnetworkv1.SriovNetworkNodeState) (bool, error) { + return false, nil } // Apply config change diff --git a/pkg/plugins/k8s/k8s_plugin.go b/pkg/plugins/k8s/k8s_plugin.go index aa28c0aa6..1a0336049 100644 --- a/pkg/plugins/k8s/k8s_plugin.go +++ b/pkg/plugins/k8s/k8s_plugin.go @@ -156,8 +156,8 @@ func (p *K8sPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeState) // TODO: implement - https://github.com/k8snetworkplumbingwg/sriov-network-operator/issues/630 // OnNodeStatusChange verify whether SriovNetworkNodeState CR status present changes on configured VFs. -func (p *K8sPlugin) CheckStatusChanges(*sriovnetworkv1.SriovNetworkNodeState) bool { - return false +func (p *K8sPlugin) CheckStatusChanges(*sriovnetworkv1.SriovNetworkNodeState) (bool, error) { + return false, nil } // Apply config change diff --git a/pkg/plugins/mellanox/mellanox_plugin.go b/pkg/plugins/mellanox/mellanox_plugin.go index 1aa1f6434..87363464b 100644 --- a/pkg/plugins/mellanox/mellanox_plugin.go +++ b/pkg/plugins/mellanox/mellanox_plugin.go @@ -173,8 +173,8 @@ func (p *MellanoxPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeS // TODO: implement - https://github.com/k8snetworkplumbingwg/sriov-network-operator/issues/631 // OnNodeStatusChange verify whether SriovNetworkNodeState CR status present changes on configured VFs. -func (p *MellanoxPlugin) CheckStatusChanges(*sriovnetworkv1.SriovNetworkNodeState) bool { - return false +func (p *MellanoxPlugin) CheckStatusChanges(*sriovnetworkv1.SriovNetworkNodeState) (bool, error) { + return false, nil } // Apply config change diff --git a/pkg/plugins/mock/mock_plugin.go b/pkg/plugins/mock/mock_plugin.go index 044d30f0e..a6ae38202 100644 --- a/pkg/plugins/mock/mock_plugin.go +++ b/pkg/plugins/mock/mock_plugin.go @@ -49,11 +49,12 @@ func (mr *MockVendorPluginMockRecorder) Apply() *gomock.Call { } // CheckStatusChanges mocks base method. -func (m *MockVendorPlugin) CheckStatusChanges(arg0 *v1.SriovNetworkNodeState) bool { +func (m *MockVendorPlugin) CheckStatusChanges(arg0 *v1.SriovNetworkNodeState) (bool, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CheckStatusChanges", arg0) ret0, _ := ret[0].(bool) - return ret0 + ret1, _ := ret[1].(error) + return ret0, ret1 } // CheckStatusChanges indicates an expected call of CheckStatusChanges. diff --git a/pkg/plugins/plugin.go b/pkg/plugins/plugin.go index 3136dfd46..830f7dd68 100644 --- a/pkg/plugins/plugin.go +++ b/pkg/plugins/plugin.go @@ -15,5 +15,5 @@ type VendorPlugin interface { // Apply config change Apply() error // CheckStatusChanges checks status changes on the SriovNetworkNodeState CR for configured VFs. - CheckStatusChanges(*sriovnetworkv1.SriovNetworkNodeState) bool + CheckStatusChanges(*sriovnetworkv1.SriovNetworkNodeState) (bool, error) } diff --git a/pkg/plugins/virtual/virtual_plugin.go b/pkg/plugins/virtual/virtual_plugin.go index cd43b120b..211cbee2c 100644 --- a/pkg/plugins/virtual/virtual_plugin.go +++ b/pkg/plugins/virtual/virtual_plugin.go @@ -68,8 +68,8 @@ func (p *VirtualPlugin) OnNodeStateChange(new *sriovnetworkv1.SriovNetworkNodeSt } // OnNodeStatusChange verify whether SriovNetworkNodeState CR status present changes on configured VFs. -func (p *VirtualPlugin) CheckStatusChanges(*sriovnetworkv1.SriovNetworkNodeState) bool { - return false +func (p *VirtualPlugin) CheckStatusChanges(*sriovnetworkv1.SriovNetworkNodeState) (bool, error) { + return false, nil } // Apply config change From 0d1a11ae9c1053d7e65af102cdef4490faf867dc Mon Sep 17 00:00:00 2001 From: Marcelo Guerrero Date: Wed, 31 Jan 2024 13:43:27 +0100 Subject: [PATCH 14/21] Add e2e test for reconciliation when status changes --- test/conformance/tests/test_sriov_operator.go | 54 +++++++++++++++++-- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/test/conformance/tests/test_sriov_operator.go b/test/conformance/tests/test_sriov_operator.go index 55711b78f..094a95cf0 100644 --- a/test/conformance/tests/test_sriov_operator.go +++ b/test/conformance/tests/test_sriov_operator.go @@ -307,8 +307,12 @@ var _ = Describe("[sriov] operator", func() { node = sriovInfos.Nodes[0] createVanillaNetworkPolicy(node, sriovInfos, numVfs, resourceName) WaitForSRIOVStable() - sriovDevice, err = sriovInfos.FindOneSriovDevice(node) + + // Update info + sriovInfos, err = cluster.DiscoverSriov(clients, operatorNamespace) Expect(err).ToNot(HaveOccurred()) + sriovDevice = findInterface(sriovInfos, node) + By("Using device " + sriovDevice.Name + " on node " + node) Eventually(func() int64 { @@ -1000,6 +1004,44 @@ var _ = Describe("[sriov] operator", func() { return runningPodB.Status.Phase }, 3*time.Minute, time.Second).Should(Equal(corev1.PodRunning)) }) + + It("should reconcile managed VF if status changes", func() { + originalMtu := sriovDevice.Mtu + newMtu := 1000 + + By("manually changing the MTU") + _, errOutput, err := runCommandOnConfigDaemon(node, "/bin/bash", "-c", fmt.Sprintf("echo %d > /sys/bus/pci/devices/%s/net/%s/mtu", newMtu, sriovDevice.PciAddress, sriovDevice.Name)) + Expect(err).ToNot(HaveOccurred()) + Expect(errOutput).To(Equal("")) + + By("waiting for the mtu to be updated in the status") + Eventually(func() sriovv1.InterfaceExts { + nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(context.Background(), node, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + return nodeState.Status.Interfaces + }, 3*time.Minute, 1*time.Second).Should(ContainElement(MatchFields( + IgnoreExtras, + Fields{ + "Name": Equal(sriovDevice.Name), + "Mtu": Equal(newMtu), + "PciAddress": Equal(sriovDevice.PciAddress), + "NumVfs": Equal(sriovDevice.NumVfs), + }))) + + By("waiting for the mtu to be restored") + Eventually(func() sriovv1.InterfaceExts { + nodeState, err := clients.SriovNetworkNodeStates(operatorNamespace).Get(context.Background(), node, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + return nodeState.Status.Interfaces + }, 3*time.Minute, 1*time.Second).Should(ContainElement(MatchFields( + IgnoreExtras, + Fields{ + "Name": Equal(sriovDevice.Name), + "Mtu": Equal(originalMtu), + "PciAddress": Equal(sriovDevice.PciAddress), + "NumVfs": Equal(sriovDevice.NumVfs), + }))) + }) }) Context("CNI Logging level", func() { @@ -2486,8 +2528,7 @@ func WaitForSRIOVStable() { }, waitingTime, 1*time.Second).Should(BeTrue()) } -func createVanillaNetworkPolicy(node string, sriovInfos *cluster.EnabledNodes, numVfs int, resourceName string) { - // For the context of tests is better to use a Mellanox card +func findInterface(sriovInfos *cluster.EnabledNodes, node string) *sriovv1.InterfaceExt { // For the context of tests is better to use a Mellanox card // as they support all the virtual function flags // if we don't find a Mellanox card we fall back to any sriov // capability interface and skip the rate limit test. @@ -2497,6 +2538,12 @@ func createVanillaNetworkPolicy(node string, sriovInfos *cluster.EnabledNodes, n Expect(err).ToNot(HaveOccurred()) } + return intf +} + +func createVanillaNetworkPolicy(node string, sriovInfos *cluster.EnabledNodes, numVfs int, resourceName string) { + intf := findInterface(sriovInfos, node) + config := &sriovv1.SriovNetworkNodePolicy{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "test-policy", @@ -2509,6 +2556,7 @@ func createVanillaNetworkPolicy(node string, sriovInfos *cluster.EnabledNodes, n }, NumVfs: numVfs, ResourceName: resourceName, + Mtu: 1500, Priority: 99, NicSelector: sriovv1.SriovNetworkNicSelector{ PfNames: []string{intf.Name}, From e675b851e33e76baa0e3f7d01e20cfacbf7dc6ef Mon Sep 17 00:00:00 2001 From: Ivan Kolodiazhnyi Date: Mon, 29 Apr 2024 22:11:17 +0300 Subject: [PATCH 15/21] Bump controller-gen to v0.14.0 Update to the newer version to omit issue [1]. [1] https://github.com/kubernetes-sigs/controller-tools/issues/888 Signed-off-by: Ivan Kolodiazhnyi --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9f4452b6d..f2c2b3b7b 100644 --- a/Makefile +++ b/Makefile @@ -142,7 +142,7 @@ mock-generate: gomock CONTROLLER_GEN = $(BIN_DIR)/controller-gen controller-gen: ## Download controller-gen locally if necessary. - $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.9.0) + $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.14.0) KUSTOMIZE = $(BIN_DIR)/kustomize kustomize: ## Download kustomize locally if necessary. From 3d9f062d21403dff35bcdf2b47e0e536fc3f8a35 Mon Sep 17 00:00:00 2001 From: Yury Kulazhenkov Date: Tue, 30 Apr 2024 12:00:55 +0300 Subject: [PATCH 16/21] Enable API package tests in CI Signed-off-by: Yury Kulazhenkov --- .github/workflows/test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 616c54053..910c1f18e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,6 +46,9 @@ jobs: - name: test cmd run: make test-cmd + - name: test api + run: make test-api + - name: test controllers on opensfhit run: CLUSTER_TYPE=openshift make test-controllers @@ -106,6 +109,9 @@ jobs: - name: test cmd run: make test-cmd + - name: test api + run: make test-api + - name: test controllers on opensfhit run: CLUSTER_TYPE=openshift make test-controllers From ec1fe2dc8fe7d0bbcf05b7b79dc0e4142c2afec3 Mon Sep 17 00:00:00 2001 From: Yury Kulazhenkov Date: Tue, 30 Apr 2024 12:02:09 +0300 Subject: [PATCH 17/21] Fix tests for api package Signed-off-by: Yury Kulazhenkov --- api/v1/testdata/TestIBRendering/simpleib.golden | 2 +- api/v1/testdata/TestRendering/chained.golden | 2 +- api/v1/testdata/TestRendering/simple.golden | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/v1/testdata/TestIBRendering/simpleib.golden b/api/v1/testdata/TestIBRendering/simpleib.golden index 747dd5253..c88d26c35 100644 --- a/api/v1/testdata/TestIBRendering/simpleib.golden +++ b/api/v1/testdata/TestIBRendering/simpleib.golden @@ -9,6 +9,6 @@ "namespace": "testnamespace" }, "spec": { - "config": "{ \"cniVersion\":\"0.3.1\", \"name\":\"\",\"type\":\"ib-sriov\",\"capabilities\":foo,\"ipam\":{} }" + "config": "{ \"cniVersion\":\"1.0.0\", \"name\":\"\",\"type\":\"ib-sriov\",\"capabilities\":foo,\"ipam\":{} }" } } diff --git a/api/v1/testdata/TestRendering/chained.golden b/api/v1/testdata/TestRendering/chained.golden index 3d1c2be4a..156acf88b 100644 --- a/api/v1/testdata/TestRendering/chained.golden +++ b/api/v1/testdata/TestRendering/chained.golden @@ -9,6 +9,6 @@ "namespace": "testnamespace" }, "spec": { - "config": "{ \"cniVersion\":\"0.3.1\", \"name\":\"\",\"plugins\": [ {\"type\":\"sriov\",\"vlan\":0,\"vlanQoS\":0,\"ipam\":{} },\n{ \"type\": \"vrf\", \"vrfname\": \"blue\" }\n] }" + "config": "{ \"cniVersion\":\"1.0.0\", \"name\":\"\",\"plugins\": [ {\"type\":\"sriov\",\"vlan\":0,\"vlanQoS\":0,\"ipam\":{} },\n{ \"type\": \"vrf\", \"vrfname\": \"blue\" }\n] }" } } diff --git a/api/v1/testdata/TestRendering/simple.golden b/api/v1/testdata/TestRendering/simple.golden index b8f94bcc4..3eb98bbb3 100644 --- a/api/v1/testdata/TestRendering/simple.golden +++ b/api/v1/testdata/TestRendering/simple.golden @@ -9,6 +9,6 @@ "namespace": "testnamespace" }, "spec": { - "config": "{ \"cniVersion\":\"0.3.1\", \"name\":\"\",\"type\":\"sriov\",\"vlan\":0,\"vlanQoS\":0,\"ipam\":{} }" + "config": "{ \"cniVersion\":\"1.0.0\", \"name\":\"\",\"type\":\"sriov\",\"vlan\":0,\"vlanQoS\":0,\"ipam\":{} }" } } From 5be2adc57409d30a57dd49ee4f120346606ad177 Mon Sep 17 00:00:00 2001 From: Sebastian Sch Date: Thu, 25 Apr 2024 17:31:50 +0300 Subject: [PATCH 18/21] Support adding or removing node Add Watch in sriovNetworkNodePolicy for adding and removing nodes. With this change we will be able act faster when new nodes are added to the cluster and not waiting for the reconcile loop requeue Signed-off-by: Sebastian Sch --- controllers/drain_controller.go | 27 ++++++++++--------- .../sriovnetworknodepolicy_controller.go | 16 +++++++++++ 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/controllers/drain_controller.go b/controllers/drain_controller.go index 2869e9a51..86da909d8 100644 --- a/controllers/drain_controller.go +++ b/controllers/drain_controller.go @@ -94,19 +94,27 @@ func (dr *DrainReconcile) Reconcile(ctx context.Context, req ctrl.Request) (ctrl // get node object node := &corev1.Node{} - err := dr.getObject(ctx, req, node) + found, err := dr.getObject(ctx, req, node) if err != nil { reqLogger.Error(err, "failed to get node object") return ctrl.Result{}, err } + if !found { + reqLogger.Info("node not found don't, requeue the request") + return ctrl.Result{}, nil + } // get sriovNodeNodeState object nodeNetworkState := &sriovnetworkv1.SriovNetworkNodeState{} - err = dr.getObject(ctx, req, nodeNetworkState) + found, err = dr.getObject(ctx, req, nodeNetworkState) if err != nil { reqLogger.Error(err, "failed to get sriovNetworkNodeState object") return ctrl.Result{}, err } + if !found { + reqLogger.Info("sriovNetworkNodeState not found, don't requeue the request") + return ctrl.Result{}, nil + } // create the drain state annotation if it doesn't exist in the sriovNetworkNodeState object nodeStateDrainAnnotationCurrent, err := dr.ensureAnnotationExists(ctx, nodeNetworkState, constants.NodeStateDrainAnnotationCurrent) @@ -241,22 +249,15 @@ func (dr *DrainReconcile) Reconcile(ctx context.Context, req ctrl.Request) (ctrl return reconcile.Result{}, fmt.Errorf("unexpected node drain annotation") } -func (dr *DrainReconcile) getObject(ctx context.Context, req ctrl.Request, object client.Object) error { - reqLogger := log.FromContext(ctx) - reqLogger.Info("getObject():") - +func (dr *DrainReconcile) getObject(ctx context.Context, req ctrl.Request, object client.Object) (bool, error) { err := dr.Get(ctx, req.NamespacedName, object) if err != nil { if errors.IsNotFound(err) { - reqLogger.Error(err, "object doesn't exist", "objectName", req.Name) - return nil + return false, nil } - - reqLogger.Error(err, "failed to get object from api re-queue the request", "objectName", req.Name) - return err + return false, err } - - return nil + return true, nil } func (dr *DrainReconcile) ensureAnnotationExists(ctx context.Context, object client.Object, key string) (string, error) { diff --git a/controllers/sriovnetworknodepolicy_controller.go b/controllers/sriovnetworknodepolicy_controller.go index c4c1fc42d..5f145d9e7 100644 --- a/controllers/sriovnetworknodepolicy_controller.go +++ b/controllers/sriovnetworknodepolicy_controller.go @@ -176,6 +176,20 @@ func (r *SriovNetworkNodePolicyReconciler) SetupWithManager(mgr ctrl.Manager) er }, } + // we want to act fast on new or deleted nodes + nodeEvenHandler := handler.Funcs{ + CreateFunc: func(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) { + log.Log.WithName("SriovNetworkNodePolicy"). + Info("Enqueuing sync for create event", "resource", e.Object.GetName()) + qHandler(q) + }, + DeleteFunc: func(ctx context.Context, e event.DeleteEvent, q workqueue.RateLimitingInterface) { + log.Log.WithName("SriovNetworkNodePolicy"). + Info("Enqueuing sync for delete event", "resource", e.Object.GetName()) + qHandler(q) + }, + } + // send initial sync event to trigger reconcile when controller is started var eventChan = make(chan event.GenericEvent, 1) eventChan <- event.GenericEvent{Object: &sriovnetworkv1.SriovNetworkNodePolicy{ @@ -184,6 +198,7 @@ func (r *SriovNetworkNodePolicyReconciler) SetupWithManager(mgr ctrl.Manager) er return ctrl.NewControllerManagedBy(mgr). For(&sriovnetworkv1.SriovNetworkNodePolicy{}). + Watches(&corev1.Node{}, nodeEvenHandler). Watches(&sriovnetworkv1.SriovNetworkNodePolicy{}, delayedEventHandler). WatchesRawSource(&source.Channel{Source: eventChan}, delayedEventHandler). Complete(r) @@ -282,6 +297,7 @@ func (r *SriovNetworkNodePolicyReconciler) syncAllSriovNetworkNodeStates(ctx con } } if !found { + logger.Info("Deleting SriovNetworkNodeState as node with that name doesn't exist", "nodeStateName", ns.Name) err := r.Delete(ctx, &ns, &client.DeleteOptions{}) if err != nil { logger.Error(err, "Fail to Delete", "SriovNetworkNodeState CR:", ns.GetName()) From 60ef85d2451790090492a6842b0a8395bd70c1e3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 03:45:51 +0000 Subject: [PATCH 19/21] Bump golang.org/x/net from 0.17.0 to 0.23.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.17.0 to 0.23.0. - [Commits](https://github.com/golang/net/compare/v0.17.0...v0.23.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 8 +- go.sum | 16 +- vendor/golang.org/x/net/context/go17.go | 1 - vendor/golang.org/x/net/context/go19.go | 1 - vendor/golang.org/x/net/context/pre_go17.go | 1 - vendor/golang.org/x/net/context/pre_go19.go | 1 - vendor/golang.org/x/net/html/token.go | 12 +- vendor/golang.org/x/net/http2/databuffer.go | 59 +-- vendor/golang.org/x/net/http2/frame.go | 42 ++- vendor/golang.org/x/net/http2/go111.go | 30 -- vendor/golang.org/x/net/http2/go115.go | 27 -- vendor/golang.org/x/net/http2/go118.go | 17 - vendor/golang.org/x/net/http2/not_go111.go | 21 -- vendor/golang.org/x/net/http2/not_go115.go | 31 -- vendor/golang.org/x/net/http2/not_go118.go | 17 - vendor/golang.org/x/net/http2/pipe.go | 11 +- vendor/golang.org/x/net/http2/server.go | 37 +- vendor/golang.org/x/net/http2/testsync.go | 331 +++++++++++++++++ vendor/golang.org/x/net/http2/transport.go | 340 ++++++++++++++---- vendor/golang.org/x/net/idna/go118.go | 1 - vendor/golang.org/x/net/idna/idna10.0.0.go | 1 - vendor/golang.org/x/net/idna/idna9.0.0.go | 1 - vendor/golang.org/x/net/idna/pre_go118.go | 1 - vendor/golang.org/x/net/idna/tables10.0.0.go | 1 - vendor/golang.org/x/net/idna/tables11.0.0.go | 1 - vendor/golang.org/x/net/idna/tables12.0.0.go | 1 - vendor/golang.org/x/net/idna/tables13.0.0.go | 1 - vendor/golang.org/x/net/idna/tables15.0.0.go | 1 - vendor/golang.org/x/net/idna/tables9.0.0.go | 1 - vendor/golang.org/x/net/idna/trie12.0.0.go | 1 - vendor/golang.org/x/net/idna/trie13.0.0.go | 1 - vendor/golang.org/x/sys/unix/aliases.go | 2 +- .../x/sys/unix/syscall_darwin_libSystem.go | 2 +- .../golang.org/x/sys/unix/syscall_freebsd.go | 12 +- vendor/golang.org/x/sys/unix/syscall_linux.go | 99 +++++ .../golang.org/x/sys/unix/zsyscall_linux.go | 10 + vendor/golang.org/x/sys/unix/ztypes_linux.go | 60 ++++ vendor/modules.txt | 10 +- 38 files changed, 900 insertions(+), 310 deletions(-) delete mode 100644 vendor/golang.org/x/net/http2/go111.go delete mode 100644 vendor/golang.org/x/net/http2/go115.go delete mode 100644 vendor/golang.org/x/net/http2/go118.go delete mode 100644 vendor/golang.org/x/net/http2/not_go111.go delete mode 100644 vendor/golang.org/x/net/http2/not_go115.go delete mode 100644 vendor/golang.org/x/net/http2/not_go118.go create mode 100644 vendor/golang.org/x/net/http2/testsync.go diff --git a/go.mod b/go.mod index 6af5986b9..fca66390c 100644 --- a/go.mod +++ b/go.mod @@ -124,14 +124,14 @@ require ( go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect go.uber.org/multierr v1.11.0 // indirect go4.org v0.0.0-20200104003542-c7e774b10ea0 // indirect - golang.org/x/crypto v0.17.0 // indirect + golang.org/x/crypto v0.21.0 // indirect golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect golang.org/x/mod v0.13.0 // indirect - golang.org/x/net v0.17.0 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.13.0 // indirect golang.org/x/sync v0.4.0 // indirect - golang.org/x/sys v0.17.0 // indirect - golang.org/x/term v0.15.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.14.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 7de2f12b5..30f10a74f 100644 --- a/go.sum +++ b/go.sum @@ -475,8 +475,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -551,8 +551,8 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -629,13 +629,13 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/vendor/golang.org/x/net/context/go17.go b/vendor/golang.org/x/net/context/go17.go index 2cb9c408f..0c1b86793 100644 --- a/vendor/golang.org/x/net/context/go17.go +++ b/vendor/golang.org/x/net/context/go17.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build go1.7 -// +build go1.7 package context diff --git a/vendor/golang.org/x/net/context/go19.go b/vendor/golang.org/x/net/context/go19.go index 64d31ecc3..e31e35a90 100644 --- a/vendor/golang.org/x/net/context/go19.go +++ b/vendor/golang.org/x/net/context/go19.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build go1.9 -// +build go1.9 package context diff --git a/vendor/golang.org/x/net/context/pre_go17.go b/vendor/golang.org/x/net/context/pre_go17.go index 7b6b68511..065ff3dfa 100644 --- a/vendor/golang.org/x/net/context/pre_go17.go +++ b/vendor/golang.org/x/net/context/pre_go17.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build !go1.7 -// +build !go1.7 package context diff --git a/vendor/golang.org/x/net/context/pre_go19.go b/vendor/golang.org/x/net/context/pre_go19.go index 1f9715341..ec5a63803 100644 --- a/vendor/golang.org/x/net/context/pre_go19.go +++ b/vendor/golang.org/x/net/context/pre_go19.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build !go1.9 -// +build !go1.9 package context diff --git a/vendor/golang.org/x/net/html/token.go b/vendor/golang.org/x/net/html/token.go index de67f938a..3c57880d6 100644 --- a/vendor/golang.org/x/net/html/token.go +++ b/vendor/golang.org/x/net/html/token.go @@ -910,9 +910,6 @@ func (z *Tokenizer) readTagAttrKey() { return } switch c { - case ' ', '\n', '\r', '\t', '\f', '/': - z.pendingAttr[0].end = z.raw.end - 1 - return case '=': if z.pendingAttr[0].start+1 == z.raw.end { // WHATWG 13.2.5.32, if we see an equals sign before the attribute name @@ -920,7 +917,9 @@ func (z *Tokenizer) readTagAttrKey() { continue } fallthrough - case '>': + case ' ', '\n', '\r', '\t', '\f', '/', '>': + // WHATWG 13.2.5.33 Attribute name state + // We need to reconsume the char in the after attribute name state to support the / character z.raw.end-- z.pendingAttr[0].end = z.raw.end return @@ -939,6 +938,11 @@ func (z *Tokenizer) readTagAttrVal() { if z.err != nil { return } + if c == '/' { + // WHATWG 13.2.5.34 After attribute name state + // U+002F SOLIDUS (/) - Switch to the self-closing start tag state. + return + } if c != '=' { z.raw.end-- return diff --git a/vendor/golang.org/x/net/http2/databuffer.go b/vendor/golang.org/x/net/http2/databuffer.go index a3067f8de..e6f55cbd1 100644 --- a/vendor/golang.org/x/net/http2/databuffer.go +++ b/vendor/golang.org/x/net/http2/databuffer.go @@ -20,41 +20,44 @@ import ( // TODO: Benchmark to determine if the pools are necessary. The GC may have // improved enough that we can instead allocate chunks like this: // make([]byte, max(16<<10, expectedBytesRemaining)) -var ( - dataChunkSizeClasses = []int{ - 1 << 10, - 2 << 10, - 4 << 10, - 8 << 10, - 16 << 10, - } - dataChunkPools = [...]sync.Pool{ - {New: func() interface{} { return make([]byte, 1<<10) }}, - {New: func() interface{} { return make([]byte, 2<<10) }}, - {New: func() interface{} { return make([]byte, 4<<10) }}, - {New: func() interface{} { return make([]byte, 8<<10) }}, - {New: func() interface{} { return make([]byte, 16<<10) }}, - } -) +var dataChunkPools = [...]sync.Pool{ + {New: func() interface{} { return new([1 << 10]byte) }}, + {New: func() interface{} { return new([2 << 10]byte) }}, + {New: func() interface{} { return new([4 << 10]byte) }}, + {New: func() interface{} { return new([8 << 10]byte) }}, + {New: func() interface{} { return new([16 << 10]byte) }}, +} func getDataBufferChunk(size int64) []byte { - i := 0 - for ; i < len(dataChunkSizeClasses)-1; i++ { - if size <= int64(dataChunkSizeClasses[i]) { - break - } + switch { + case size <= 1<<10: + return dataChunkPools[0].Get().(*[1 << 10]byte)[:] + case size <= 2<<10: + return dataChunkPools[1].Get().(*[2 << 10]byte)[:] + case size <= 4<<10: + return dataChunkPools[2].Get().(*[4 << 10]byte)[:] + case size <= 8<<10: + return dataChunkPools[3].Get().(*[8 << 10]byte)[:] + default: + return dataChunkPools[4].Get().(*[16 << 10]byte)[:] } - return dataChunkPools[i].Get().([]byte) } func putDataBufferChunk(p []byte) { - for i, n := range dataChunkSizeClasses { - if len(p) == n { - dataChunkPools[i].Put(p) - return - } + switch len(p) { + case 1 << 10: + dataChunkPools[0].Put((*[1 << 10]byte)(p)) + case 2 << 10: + dataChunkPools[1].Put((*[2 << 10]byte)(p)) + case 4 << 10: + dataChunkPools[2].Put((*[4 << 10]byte)(p)) + case 8 << 10: + dataChunkPools[3].Put((*[8 << 10]byte)(p)) + case 16 << 10: + dataChunkPools[4].Put((*[16 << 10]byte)(p)) + default: + panic(fmt.Sprintf("unexpected buffer len=%v", len(p))) } - panic(fmt.Sprintf("unexpected buffer len=%v", len(p))) } // dataBuffer is an io.ReadWriter backed by a list of data chunks. diff --git a/vendor/golang.org/x/net/http2/frame.go b/vendor/golang.org/x/net/http2/frame.go index c1f6b90dc..43557ab7e 100644 --- a/vendor/golang.org/x/net/http2/frame.go +++ b/vendor/golang.org/x/net/http2/frame.go @@ -1510,13 +1510,12 @@ func (mh *MetaHeadersFrame) checkPseudos() error { } func (fr *Framer) maxHeaderStringLen() int { - v := fr.maxHeaderListSize() - if uint32(int(v)) == v { - return int(v) + v := int(fr.maxHeaderListSize()) + if v < 0 { + // If maxHeaderListSize overflows an int, use no limit (0). + return 0 } - // They had a crazy big number for MaxHeaderBytes anyway, - // so give them unlimited header lengths: - return 0 + return v } // readMetaFrame returns 0 or more CONTINUATION frames from fr and @@ -1565,6 +1564,7 @@ func (fr *Framer) readMetaFrame(hf *HeadersFrame) (*MetaHeadersFrame, error) { if size > remainSize { hdec.SetEmitEnabled(false) mh.Truncated = true + remainSize = 0 return } remainSize -= size @@ -1577,6 +1577,36 @@ func (fr *Framer) readMetaFrame(hf *HeadersFrame) (*MetaHeadersFrame, error) { var hc headersOrContinuation = hf for { frag := hc.HeaderBlockFragment() + + // Avoid parsing large amounts of headers that we will then discard. + // If the sender exceeds the max header list size by too much, + // skip parsing the fragment and close the connection. + // + // "Too much" is either any CONTINUATION frame after we've already + // exceeded the max header list size (in which case remainSize is 0), + // or a frame whose encoded size is more than twice the remaining + // header list bytes we're willing to accept. + if int64(len(frag)) > int64(2*remainSize) { + if VerboseLogs { + log.Printf("http2: header list too large") + } + // It would be nice to send a RST_STREAM before sending the GOAWAY, + // but the structure of the server's frame writer makes this difficult. + return nil, ConnectionError(ErrCodeProtocol) + } + + // Also close the connection after any CONTINUATION frame following an + // invalid header, since we stop tracking the size of the headers after + // an invalid one. + if invalid != nil { + if VerboseLogs { + log.Printf("http2: invalid header: %v", invalid) + } + // It would be nice to send a RST_STREAM before sending the GOAWAY, + // but the structure of the server's frame writer makes this difficult. + return nil, ConnectionError(ErrCodeProtocol) + } + if _, err := hdec.Write(frag); err != nil { return nil, ConnectionError(ErrCodeCompression) } diff --git a/vendor/golang.org/x/net/http2/go111.go b/vendor/golang.org/x/net/http2/go111.go deleted file mode 100644 index 5bf62b032..000000000 --- a/vendor/golang.org/x/net/http2/go111.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build go1.11 -// +build go1.11 - -package http2 - -import ( - "net/http/httptrace" - "net/textproto" -) - -func traceHasWroteHeaderField(trace *httptrace.ClientTrace) bool { - return trace != nil && trace.WroteHeaderField != nil -} - -func traceWroteHeaderField(trace *httptrace.ClientTrace, k, v string) { - if trace != nil && trace.WroteHeaderField != nil { - trace.WroteHeaderField(k, []string{v}) - } -} - -func traceGot1xxResponseFunc(trace *httptrace.ClientTrace) func(int, textproto.MIMEHeader) error { - if trace != nil { - return trace.Got1xxResponse - } - return nil -} diff --git a/vendor/golang.org/x/net/http2/go115.go b/vendor/golang.org/x/net/http2/go115.go deleted file mode 100644 index 908af1ab9..000000000 --- a/vendor/golang.org/x/net/http2/go115.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2021 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build go1.15 -// +build go1.15 - -package http2 - -import ( - "context" - "crypto/tls" -) - -// dialTLSWithContext uses tls.Dialer, added in Go 1.15, to open a TLS -// connection. -func (t *Transport) dialTLSWithContext(ctx context.Context, network, addr string, cfg *tls.Config) (*tls.Conn, error) { - dialer := &tls.Dialer{ - Config: cfg, - } - cn, err := dialer.DialContext(ctx, network, addr) - if err != nil { - return nil, err - } - tlsCn := cn.(*tls.Conn) // DialContext comment promises this will always succeed - return tlsCn, nil -} diff --git a/vendor/golang.org/x/net/http2/go118.go b/vendor/golang.org/x/net/http2/go118.go deleted file mode 100644 index aca4b2b31..000000000 --- a/vendor/golang.org/x/net/http2/go118.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2021 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build go1.18 -// +build go1.18 - -package http2 - -import ( - "crypto/tls" - "net" -) - -func tlsUnderlyingConn(tc *tls.Conn) net.Conn { - return tc.NetConn() -} diff --git a/vendor/golang.org/x/net/http2/not_go111.go b/vendor/golang.org/x/net/http2/not_go111.go deleted file mode 100644 index cc0baa819..000000000 --- a/vendor/golang.org/x/net/http2/not_go111.go +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !go1.11 -// +build !go1.11 - -package http2 - -import ( - "net/http/httptrace" - "net/textproto" -) - -func traceHasWroteHeaderField(trace *httptrace.ClientTrace) bool { return false } - -func traceWroteHeaderField(trace *httptrace.ClientTrace, k, v string) {} - -func traceGot1xxResponseFunc(trace *httptrace.ClientTrace) func(int, textproto.MIMEHeader) error { - return nil -} diff --git a/vendor/golang.org/x/net/http2/not_go115.go b/vendor/golang.org/x/net/http2/not_go115.go deleted file mode 100644 index e6c04cf7a..000000000 --- a/vendor/golang.org/x/net/http2/not_go115.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2021 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !go1.15 -// +build !go1.15 - -package http2 - -import ( - "context" - "crypto/tls" -) - -// dialTLSWithContext opens a TLS connection. -func (t *Transport) dialTLSWithContext(ctx context.Context, network, addr string, cfg *tls.Config) (*tls.Conn, error) { - cn, err := tls.Dial(network, addr, cfg) - if err != nil { - return nil, err - } - if err := cn.Handshake(); err != nil { - return nil, err - } - if cfg.InsecureSkipVerify { - return cn, nil - } - if err := cn.VerifyHostname(cfg.ServerName); err != nil { - return nil, err - } - return cn, nil -} diff --git a/vendor/golang.org/x/net/http2/not_go118.go b/vendor/golang.org/x/net/http2/not_go118.go deleted file mode 100644 index eab532c96..000000000 --- a/vendor/golang.org/x/net/http2/not_go118.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2021 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !go1.18 -// +build !go1.18 - -package http2 - -import ( - "crypto/tls" - "net" -) - -func tlsUnderlyingConn(tc *tls.Conn) net.Conn { - return nil -} diff --git a/vendor/golang.org/x/net/http2/pipe.go b/vendor/golang.org/x/net/http2/pipe.go index 684d984fd..3b9f06b96 100644 --- a/vendor/golang.org/x/net/http2/pipe.go +++ b/vendor/golang.org/x/net/http2/pipe.go @@ -77,7 +77,10 @@ func (p *pipe) Read(d []byte) (n int, err error) { } } -var errClosedPipeWrite = errors.New("write on closed buffer") +var ( + errClosedPipeWrite = errors.New("write on closed buffer") + errUninitializedPipeWrite = errors.New("write on uninitialized buffer") +) // Write copies bytes from p into the buffer and wakes a reader. // It is an error to write more data than the buffer can hold. @@ -91,6 +94,12 @@ func (p *pipe) Write(d []byte) (n int, err error) { if p.err != nil || p.breakErr != nil { return 0, errClosedPipeWrite } + // pipe.setBuffer is never invoked, leaving the buffer uninitialized. + // We shouldn't try to write to an uninitialized pipe, + // but returning an error is better than panicking. + if p.b == nil { + return 0, errUninitializedPipeWrite + } return p.b.Write(d) } diff --git a/vendor/golang.org/x/net/http2/server.go b/vendor/golang.org/x/net/http2/server.go index 02c88b6b3..ce2e8b40e 100644 --- a/vendor/golang.org/x/net/http2/server.go +++ b/vendor/golang.org/x/net/http2/server.go @@ -124,6 +124,7 @@ type Server struct { // IdleTimeout specifies how long until idle clients should be // closed with a GOAWAY frame. PING frames are not considered // activity for the purposes of IdleTimeout. + // If zero or negative, there is no timeout. IdleTimeout time.Duration // MaxUploadBufferPerConnection is the size of the initial flow @@ -434,7 +435,7 @@ func (s *Server) ServeConn(c net.Conn, opts *ServeConnOpts) { // passes the connection off to us with the deadline already set. // Write deadlines are set per stream in serverConn.newStream. // Disarm the net.Conn write deadline here. - if sc.hs.WriteTimeout != 0 { + if sc.hs.WriteTimeout > 0 { sc.conn.SetWriteDeadline(time.Time{}) } @@ -924,7 +925,7 @@ func (sc *serverConn) serve() { sc.setConnState(http.StateActive) sc.setConnState(http.StateIdle) - if sc.srv.IdleTimeout != 0 { + if sc.srv.IdleTimeout > 0 { sc.idleTimer = time.AfterFunc(sc.srv.IdleTimeout, sc.onIdleTimer) defer sc.idleTimer.Stop() } @@ -1637,7 +1638,7 @@ func (sc *serverConn) closeStream(st *stream, err error) { delete(sc.streams, st.id) if len(sc.streams) == 0 { sc.setConnState(http.StateIdle) - if sc.srv.IdleTimeout != 0 { + if sc.srv.IdleTimeout > 0 { sc.idleTimer.Reset(sc.srv.IdleTimeout) } if h1ServerKeepAlivesDisabled(sc.hs) { @@ -2017,7 +2018,7 @@ func (sc *serverConn) processHeaders(f *MetaHeadersFrame) error { // similar to how the http1 server works. Here it's // technically more like the http1 Server's ReadHeaderTimeout // (in Go 1.8), though. That's a more sane option anyway. - if sc.hs.ReadTimeout != 0 { + if sc.hs.ReadTimeout > 0 { sc.conn.SetReadDeadline(time.Time{}) st.readDeadline = time.AfterFunc(sc.hs.ReadTimeout, st.onReadTimeout) } @@ -2038,7 +2039,7 @@ func (sc *serverConn) upgradeRequest(req *http.Request) { // Disable any read deadline set by the net/http package // prior to the upgrade. - if sc.hs.ReadTimeout != 0 { + if sc.hs.ReadTimeout > 0 { sc.conn.SetReadDeadline(time.Time{}) } @@ -2116,7 +2117,7 @@ func (sc *serverConn) newStream(id, pusherID uint32, state streamState) *stream st.flow.conn = &sc.flow // link to conn-level counter st.flow.add(sc.initialStreamSendWindowSize) st.inflow.init(sc.srv.initialStreamRecvWindowSize()) - if sc.hs.WriteTimeout != 0 { + if sc.hs.WriteTimeout > 0 { st.writeDeadline = time.AfterFunc(sc.hs.WriteTimeout, st.onWriteTimeout) } @@ -2549,7 +2550,6 @@ type responseWriterState struct { wroteHeader bool // WriteHeader called (explicitly or implicitly). Not necessarily sent to user yet. sentHeader bool // have we sent the header frame? handlerDone bool // handler has finished - dirty bool // a Write failed; don't reuse this responseWriterState sentContentLen int64 // non-zero if handler set a Content-Length header wroteBytes int64 @@ -2669,7 +2669,6 @@ func (rws *responseWriterState) writeChunk(p []byte) (n int, err error) { date: date, }) if err != nil { - rws.dirty = true return 0, err } if endStream { @@ -2690,7 +2689,6 @@ func (rws *responseWriterState) writeChunk(p []byte) (n int, err error) { if len(p) > 0 || endStream { // only send a 0 byte DATA frame if we're ending the stream. if err := rws.conn.writeDataFromHandler(rws.stream, p, endStream); err != nil { - rws.dirty = true return 0, err } } @@ -2702,9 +2700,6 @@ func (rws *responseWriterState) writeChunk(p []byte) (n int, err error) { trailers: rws.trailers, endStream: true, }) - if err != nil { - rws.dirty = true - } return len(p), err } return len(p), nil @@ -2920,14 +2915,12 @@ func (rws *responseWriterState) writeHeader(code int) { h.Del("Transfer-Encoding") } - if rws.conn.writeHeaders(rws.stream, &writeResHeaders{ + rws.conn.writeHeaders(rws.stream, &writeResHeaders{ streamID: rws.stream.id, httpResCode: code, h: h, endStream: rws.handlerDone && !rws.hasTrailers(), - }) != nil { - rws.dirty = true - } + }) return } @@ -2992,19 +2985,10 @@ func (w *responseWriter) write(lenData int, dataB []byte, dataS string) (n int, func (w *responseWriter) handlerDone() { rws := w.rws - dirty := rws.dirty rws.handlerDone = true w.Flush() w.rws = nil - if !dirty { - // Only recycle the pool if all prior Write calls to - // the serverConn goroutine completed successfully. If - // they returned earlier due to resets from the peer - // there might still be write goroutines outstanding - // from the serverConn referencing the rws memory. See - // issue 20704. - responseWriterStatePool.Put(rws) - } + responseWriterStatePool.Put(rws) } // Push errors. @@ -3187,6 +3171,7 @@ func (sc *serverConn) startPush(msg *startPushRequest) { panic(fmt.Sprintf("newWriterAndRequestNoBody(%+v): %v", msg.url, err)) } + sc.curHandlers++ go sc.runHandler(rw, req, sc.handler.ServeHTTP) return promisedID, nil } diff --git a/vendor/golang.org/x/net/http2/testsync.go b/vendor/golang.org/x/net/http2/testsync.go new file mode 100644 index 000000000..61075bd16 --- /dev/null +++ b/vendor/golang.org/x/net/http2/testsync.go @@ -0,0 +1,331 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +package http2 + +import ( + "context" + "sync" + "time" +) + +// testSyncHooks coordinates goroutines in tests. +// +// For example, a call to ClientConn.RoundTrip involves several goroutines, including: +// - the goroutine running RoundTrip; +// - the clientStream.doRequest goroutine, which writes the request; and +// - the clientStream.readLoop goroutine, which reads the response. +// +// Using testSyncHooks, a test can start a RoundTrip and identify when all these goroutines +// are blocked waiting for some condition such as reading the Request.Body or waiting for +// flow control to become available. +// +// The testSyncHooks also manage timers and synthetic time in tests. +// This permits us to, for example, start a request and cause it to time out waiting for +// response headers without resorting to time.Sleep calls. +type testSyncHooks struct { + // active/inactive act as a mutex and condition variable. + // + // - neither chan contains a value: testSyncHooks is locked. + // - active contains a value: unlocked, and at least one goroutine is not blocked + // - inactive contains a value: unlocked, and all goroutines are blocked + active chan struct{} + inactive chan struct{} + + // goroutine counts + total int // total goroutines + condwait map[*sync.Cond]int // blocked in sync.Cond.Wait + blocked []*testBlockedGoroutine // otherwise blocked + + // fake time + now time.Time + timers []*fakeTimer + + // Transport testing: Report various events. + newclientconn func(*ClientConn) + newstream func(*clientStream) +} + +// testBlockedGoroutine is a blocked goroutine. +type testBlockedGoroutine struct { + f func() bool // blocked until f returns true + ch chan struct{} // closed when unblocked +} + +func newTestSyncHooks() *testSyncHooks { + h := &testSyncHooks{ + active: make(chan struct{}, 1), + inactive: make(chan struct{}, 1), + condwait: map[*sync.Cond]int{}, + } + h.inactive <- struct{}{} + h.now = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) + return h +} + +// lock acquires the testSyncHooks mutex. +func (h *testSyncHooks) lock() { + select { + case <-h.active: + case <-h.inactive: + } +} + +// waitInactive waits for all goroutines to become inactive. +func (h *testSyncHooks) waitInactive() { + for { + <-h.inactive + if !h.unlock() { + break + } + } +} + +// unlock releases the testSyncHooks mutex. +// It reports whether any goroutines are active. +func (h *testSyncHooks) unlock() (active bool) { + // Look for a blocked goroutine which can be unblocked. + blocked := h.blocked[:0] + unblocked := false + for _, b := range h.blocked { + if !unblocked && b.f() { + unblocked = true + close(b.ch) + } else { + blocked = append(blocked, b) + } + } + h.blocked = blocked + + // Count goroutines blocked on condition variables. + condwait := 0 + for _, count := range h.condwait { + condwait += count + } + + if h.total > condwait+len(blocked) { + h.active <- struct{}{} + return true + } else { + h.inactive <- struct{}{} + return false + } +} + +// goRun starts a new goroutine. +func (h *testSyncHooks) goRun(f func()) { + h.lock() + h.total++ + h.unlock() + go func() { + defer func() { + h.lock() + h.total-- + h.unlock() + }() + f() + }() +} + +// blockUntil indicates that a goroutine is blocked waiting for some condition to become true. +// It waits until f returns true before proceeding. +// +// Example usage: +// +// h.blockUntil(func() bool { +// // Is the context done yet? +// select { +// case <-ctx.Done(): +// default: +// return false +// } +// return true +// }) +// // Wait for the context to become done. +// <-ctx.Done() +// +// The function f passed to blockUntil must be non-blocking and idempotent. +func (h *testSyncHooks) blockUntil(f func() bool) { + if f() { + return + } + ch := make(chan struct{}) + h.lock() + h.blocked = append(h.blocked, &testBlockedGoroutine{ + f: f, + ch: ch, + }) + h.unlock() + <-ch +} + +// broadcast is sync.Cond.Broadcast. +func (h *testSyncHooks) condBroadcast(cond *sync.Cond) { + h.lock() + delete(h.condwait, cond) + h.unlock() + cond.Broadcast() +} + +// broadcast is sync.Cond.Wait. +func (h *testSyncHooks) condWait(cond *sync.Cond) { + h.lock() + h.condwait[cond]++ + h.unlock() +} + +// newTimer creates a new fake timer. +func (h *testSyncHooks) newTimer(d time.Duration) timer { + h.lock() + defer h.unlock() + t := &fakeTimer{ + hooks: h, + when: h.now.Add(d), + c: make(chan time.Time), + } + h.timers = append(h.timers, t) + return t +} + +// afterFunc creates a new fake AfterFunc timer. +func (h *testSyncHooks) afterFunc(d time.Duration, f func()) timer { + h.lock() + defer h.unlock() + t := &fakeTimer{ + hooks: h, + when: h.now.Add(d), + f: f, + } + h.timers = append(h.timers, t) + return t +} + +func (h *testSyncHooks) contextWithTimeout(ctx context.Context, d time.Duration) (context.Context, context.CancelFunc) { + ctx, cancel := context.WithCancel(ctx) + t := h.afterFunc(d, cancel) + return ctx, func() { + t.Stop() + cancel() + } +} + +func (h *testSyncHooks) timeUntilEvent() time.Duration { + h.lock() + defer h.unlock() + var next time.Time + for _, t := range h.timers { + if next.IsZero() || t.when.Before(next) { + next = t.when + } + } + if d := next.Sub(h.now); d > 0 { + return d + } + return 0 +} + +// advance advances time and causes synthetic timers to fire. +func (h *testSyncHooks) advance(d time.Duration) { + h.lock() + defer h.unlock() + h.now = h.now.Add(d) + timers := h.timers[:0] + for _, t := range h.timers { + t := t // remove after go.mod depends on go1.22 + t.mu.Lock() + switch { + case t.when.After(h.now): + timers = append(timers, t) + case t.when.IsZero(): + // stopped timer + default: + t.when = time.Time{} + if t.c != nil { + close(t.c) + } + if t.f != nil { + h.total++ + go func() { + defer func() { + h.lock() + h.total-- + h.unlock() + }() + t.f() + }() + } + } + t.mu.Unlock() + } + h.timers = timers +} + +// A timer wraps a time.Timer, or a synthetic equivalent in tests. +// Unlike time.Timer, timer is single-use: The timer channel is closed when the timer expires. +type timer interface { + C() <-chan time.Time + Stop() bool + Reset(d time.Duration) bool +} + +// timeTimer implements timer using real time. +type timeTimer struct { + t *time.Timer + c chan time.Time +} + +// newTimeTimer creates a new timer using real time. +func newTimeTimer(d time.Duration) timer { + ch := make(chan time.Time) + t := time.AfterFunc(d, func() { + close(ch) + }) + return &timeTimer{t, ch} +} + +// newTimeAfterFunc creates an AfterFunc timer using real time. +func newTimeAfterFunc(d time.Duration, f func()) timer { + return &timeTimer{ + t: time.AfterFunc(d, f), + } +} + +func (t timeTimer) C() <-chan time.Time { return t.c } +func (t timeTimer) Stop() bool { return t.t.Stop() } +func (t timeTimer) Reset(d time.Duration) bool { return t.t.Reset(d) } + +// fakeTimer implements timer using fake time. +type fakeTimer struct { + hooks *testSyncHooks + + mu sync.Mutex + when time.Time // when the timer will fire + c chan time.Time // closed when the timer fires; mutually exclusive with f + f func() // called when the timer fires; mutually exclusive with c +} + +func (t *fakeTimer) C() <-chan time.Time { return t.c } + +func (t *fakeTimer) Stop() bool { + t.mu.Lock() + defer t.mu.Unlock() + stopped := t.when.IsZero() + t.when = time.Time{} + return stopped +} + +func (t *fakeTimer) Reset(d time.Duration) bool { + if t.c != nil || t.f == nil { + panic("fakeTimer only supports Reset on AfterFunc timers") + } + t.mu.Lock() + defer t.mu.Unlock() + t.hooks.lock() + defer t.hooks.unlock() + active := !t.when.IsZero() + t.when = t.hooks.now.Add(d) + if !active { + t.hooks.timers = append(t.hooks.timers, t) + } + return active +} diff --git a/vendor/golang.org/x/net/http2/transport.go b/vendor/golang.org/x/net/http2/transport.go index 4515b22c4..ce375c8c7 100644 --- a/vendor/golang.org/x/net/http2/transport.go +++ b/vendor/golang.org/x/net/http2/transport.go @@ -147,6 +147,12 @@ type Transport struct { // waiting for their turn. StrictMaxConcurrentStreams bool + // IdleConnTimeout is the maximum amount of time an idle + // (keep-alive) connection will remain idle before closing + // itself. + // Zero means no limit. + IdleConnTimeout time.Duration + // ReadIdleTimeout is the timeout after which a health check using ping // frame will be carried out if no frame is received on the connection. // Note that a ping response will is considered a received frame, so if @@ -178,6 +184,8 @@ type Transport struct { connPoolOnce sync.Once connPoolOrDef ClientConnPool // non-nil version of ConnPool + + syncHooks *testSyncHooks } func (t *Transport) maxHeaderListSize() uint32 { @@ -302,7 +310,7 @@ type ClientConn struct { readerErr error // set before readerDone is closed idleTimeout time.Duration // or 0 for never - idleTimer *time.Timer + idleTimer timer mu sync.Mutex // guards following cond *sync.Cond // hold mu; broadcast on flow/closed changes @@ -344,6 +352,60 @@ type ClientConn struct { werr error // first write error that has occurred hbuf bytes.Buffer // HPACK encoder writes into this henc *hpack.Encoder + + syncHooks *testSyncHooks // can be nil +} + +// Hook points used for testing. +// Outside of tests, cc.syncHooks is nil and these all have minimal implementations. +// Inside tests, see the testSyncHooks function docs. + +// goRun starts a new goroutine. +func (cc *ClientConn) goRun(f func()) { + if cc.syncHooks != nil { + cc.syncHooks.goRun(f) + return + } + go f() +} + +// condBroadcast is cc.cond.Broadcast. +func (cc *ClientConn) condBroadcast() { + if cc.syncHooks != nil { + cc.syncHooks.condBroadcast(cc.cond) + } + cc.cond.Broadcast() +} + +// condWait is cc.cond.Wait. +func (cc *ClientConn) condWait() { + if cc.syncHooks != nil { + cc.syncHooks.condWait(cc.cond) + } + cc.cond.Wait() +} + +// newTimer creates a new time.Timer, or a synthetic timer in tests. +func (cc *ClientConn) newTimer(d time.Duration) timer { + if cc.syncHooks != nil { + return cc.syncHooks.newTimer(d) + } + return newTimeTimer(d) +} + +// afterFunc creates a new time.AfterFunc timer, or a synthetic timer in tests. +func (cc *ClientConn) afterFunc(d time.Duration, f func()) timer { + if cc.syncHooks != nil { + return cc.syncHooks.afterFunc(d, f) + } + return newTimeAfterFunc(d, f) +} + +func (cc *ClientConn) contextWithTimeout(ctx context.Context, d time.Duration) (context.Context, context.CancelFunc) { + if cc.syncHooks != nil { + return cc.syncHooks.contextWithTimeout(ctx, d) + } + return context.WithTimeout(ctx, d) } // clientStream is the state for a single HTTP/2 stream. One of these @@ -425,7 +487,7 @@ func (cs *clientStream) abortStreamLocked(err error) { // TODO(dneil): Clean up tests where cs.cc.cond is nil. if cs.cc.cond != nil { // Wake up writeRequestBody if it is waiting on flow control. - cs.cc.cond.Broadcast() + cs.cc.condBroadcast() } } @@ -435,7 +497,7 @@ func (cs *clientStream) abortRequestBodyWrite() { defer cc.mu.Unlock() if cs.reqBody != nil && cs.reqBodyClosed == nil { cs.closeReqBodyLocked() - cc.cond.Broadcast() + cc.condBroadcast() } } @@ -445,10 +507,10 @@ func (cs *clientStream) closeReqBodyLocked() { } cs.reqBodyClosed = make(chan struct{}) reqBodyClosed := cs.reqBodyClosed - go func() { + cs.cc.goRun(func() { cs.reqBody.Close() close(reqBodyClosed) - }() + }) } type stickyErrWriter struct { @@ -537,15 +599,6 @@ func authorityAddr(scheme string, authority string) (addr string) { return net.JoinHostPort(host, port) } -var retryBackoffHook func(time.Duration) *time.Timer - -func backoffNewTimer(d time.Duration) *time.Timer { - if retryBackoffHook != nil { - return retryBackoffHook(d) - } - return time.NewTimer(d) -} - // RoundTripOpt is like RoundTrip, but takes options. func (t *Transport) RoundTripOpt(req *http.Request, opt RoundTripOpt) (*http.Response, error) { if !(req.URL.Scheme == "https" || (req.URL.Scheme == "http" && t.AllowHTTP)) { @@ -573,13 +626,27 @@ func (t *Transport) RoundTripOpt(req *http.Request, opt RoundTripOpt) (*http.Res backoff := float64(uint(1) << (uint(retry) - 1)) backoff += backoff * (0.1 * mathrand.Float64()) d := time.Second * time.Duration(backoff) - timer := backoffNewTimer(d) + var tm timer + if t.syncHooks != nil { + tm = t.syncHooks.newTimer(d) + t.syncHooks.blockUntil(func() bool { + select { + case <-tm.C(): + case <-req.Context().Done(): + default: + return false + } + return true + }) + } else { + tm = newTimeTimer(d) + } select { - case <-timer.C: + case <-tm.C(): t.vlogf("RoundTrip retrying after failure: %v", roundTripErr) continue case <-req.Context().Done(): - timer.Stop() + tm.Stop() err = req.Context().Err() } } @@ -658,6 +725,9 @@ func canRetryError(err error) bool { } func (t *Transport) dialClientConn(ctx context.Context, addr string, singleUse bool) (*ClientConn, error) { + if t.syncHooks != nil { + return t.newClientConn(nil, singleUse, t.syncHooks) + } host, _, err := net.SplitHostPort(addr) if err != nil { return nil, err @@ -666,7 +736,7 @@ func (t *Transport) dialClientConn(ctx context.Context, addr string, singleUse b if err != nil { return nil, err } - return t.newClientConn(tconn, singleUse) + return t.newClientConn(tconn, singleUse, nil) } func (t *Transport) newTLSConfig(host string) *tls.Config { @@ -732,10 +802,10 @@ func (t *Transport) maxEncoderHeaderTableSize() uint32 { } func (t *Transport) NewClientConn(c net.Conn) (*ClientConn, error) { - return t.newClientConn(c, t.disableKeepAlives()) + return t.newClientConn(c, t.disableKeepAlives(), nil) } -func (t *Transport) newClientConn(c net.Conn, singleUse bool) (*ClientConn, error) { +func (t *Transport) newClientConn(c net.Conn, singleUse bool, hooks *testSyncHooks) (*ClientConn, error) { cc := &ClientConn{ t: t, tconn: c, @@ -750,10 +820,15 @@ func (t *Transport) newClientConn(c net.Conn, singleUse bool) (*ClientConn, erro wantSettingsAck: true, pings: make(map[[8]byte]chan struct{}), reqHeaderMu: make(chan struct{}, 1), + syncHooks: hooks, + } + if hooks != nil { + hooks.newclientconn(cc) + c = cc.tconn } if d := t.idleConnTimeout(); d != 0 { cc.idleTimeout = d - cc.idleTimer = time.AfterFunc(d, cc.onIdleTimeout) + cc.idleTimer = cc.afterFunc(d, cc.onIdleTimeout) } if VerboseLogs { t.vlogf("http2: Transport creating client conn %p to %v", cc, c.RemoteAddr()) @@ -818,7 +893,7 @@ func (t *Transport) newClientConn(c net.Conn, singleUse bool) (*ClientConn, erro return nil, cc.werr } - go cc.readLoop() + cc.goRun(cc.readLoop) return cc, nil } @@ -826,7 +901,7 @@ func (cc *ClientConn) healthCheck() { pingTimeout := cc.t.pingTimeout() // We don't need to periodically ping in the health check, because the readLoop of ClientConn will // trigger the healthCheck again if there is no frame received. - ctx, cancel := context.WithTimeout(context.Background(), pingTimeout) + ctx, cancel := cc.contextWithTimeout(context.Background(), pingTimeout) defer cancel() cc.vlogf("http2: Transport sending health check") err := cc.Ping(ctx) @@ -1018,7 +1093,7 @@ func (cc *ClientConn) forceCloseConn() { if !ok { return } - if nc := tlsUnderlyingConn(tc); nc != nil { + if nc := tc.NetConn(); nc != nil { nc.Close() } } @@ -1056,7 +1131,7 @@ func (cc *ClientConn) Shutdown(ctx context.Context) error { // Wait for all in-flight streams to complete or connection to close done := make(chan struct{}) cancelled := false // guarded by cc.mu - go func() { + cc.goRun(func() { cc.mu.Lock() defer cc.mu.Unlock() for { @@ -1068,9 +1143,9 @@ func (cc *ClientConn) Shutdown(ctx context.Context) error { if cancelled { break } - cc.cond.Wait() + cc.condWait() } - }() + }) shutdownEnterWaitStateHook() select { case <-done: @@ -1080,7 +1155,7 @@ func (cc *ClientConn) Shutdown(ctx context.Context) error { cc.mu.Lock() // Free the goroutine above cancelled = true - cc.cond.Broadcast() + cc.condBroadcast() cc.mu.Unlock() return ctx.Err() } @@ -1118,7 +1193,7 @@ func (cc *ClientConn) closeForError(err error) { for _, cs := range cc.streams { cs.abortStreamLocked(err) } - cc.cond.Broadcast() + cc.condBroadcast() cc.mu.Unlock() cc.closeConn() } @@ -1215,6 +1290,10 @@ func (cc *ClientConn) decrStreamReservationsLocked() { } func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { + return cc.roundTrip(req, nil) +} + +func (cc *ClientConn) roundTrip(req *http.Request, streamf func(*clientStream)) (*http.Response, error) { ctx := req.Context() cs := &clientStream{ cc: cc, @@ -1229,9 +1308,23 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { respHeaderRecv: make(chan struct{}), donec: make(chan struct{}), } - go cs.doRequest(req) + cc.goRun(func() { + cs.doRequest(req) + }) waitDone := func() error { + if cc.syncHooks != nil { + cc.syncHooks.blockUntil(func() bool { + select { + case <-cs.donec: + case <-ctx.Done(): + case <-cs.reqCancel: + default: + return false + } + return true + }) + } select { case <-cs.donec: return nil @@ -1292,7 +1385,24 @@ func (cc *ClientConn) RoundTrip(req *http.Request) (*http.Response, error) { return err } + if streamf != nil { + streamf(cs) + } + for { + if cc.syncHooks != nil { + cc.syncHooks.blockUntil(func() bool { + select { + case <-cs.respHeaderRecv: + case <-cs.abort: + case <-ctx.Done(): + case <-cs.reqCancel: + default: + return false + } + return true + }) + } select { case <-cs.respHeaderRecv: return handleResponseHeaders() @@ -1348,6 +1458,21 @@ func (cs *clientStream) writeRequest(req *http.Request) (err error) { if cc.reqHeaderMu == nil { panic("RoundTrip on uninitialized ClientConn") // for tests } + var newStreamHook func(*clientStream) + if cc.syncHooks != nil { + newStreamHook = cc.syncHooks.newstream + cc.syncHooks.blockUntil(func() bool { + select { + case cc.reqHeaderMu <- struct{}{}: + <-cc.reqHeaderMu + case <-cs.reqCancel: + case <-ctx.Done(): + default: + return false + } + return true + }) + } select { case cc.reqHeaderMu <- struct{}{}: case <-cs.reqCancel: @@ -1372,6 +1497,10 @@ func (cs *clientStream) writeRequest(req *http.Request) (err error) { } cc.mu.Unlock() + if newStreamHook != nil { + newStreamHook(cs) + } + // TODO(bradfitz): this is a copy of the logic in net/http. Unify somewhere? if !cc.t.disableCompression() && req.Header.Get("Accept-Encoding") == "" && @@ -1452,15 +1581,30 @@ func (cs *clientStream) writeRequest(req *http.Request) (err error) { var respHeaderTimer <-chan time.Time var respHeaderRecv chan struct{} if d := cc.responseHeaderTimeout(); d != 0 { - timer := time.NewTimer(d) + timer := cc.newTimer(d) defer timer.Stop() - respHeaderTimer = timer.C + respHeaderTimer = timer.C() respHeaderRecv = cs.respHeaderRecv } // Wait until the peer half-closes its end of the stream, // or until the request is aborted (via context, error, or otherwise), // whichever comes first. for { + if cc.syncHooks != nil { + cc.syncHooks.blockUntil(func() bool { + select { + case <-cs.peerClosed: + case <-respHeaderTimer: + case <-respHeaderRecv: + case <-cs.abort: + case <-ctx.Done(): + case <-cs.reqCancel: + default: + return false + } + return true + }) + } select { case <-cs.peerClosed: return nil @@ -1609,7 +1753,7 @@ func (cc *ClientConn) awaitOpenSlotForStreamLocked(cs *clientStream) error { return nil } cc.pendingRequests++ - cc.cond.Wait() + cc.condWait() cc.pendingRequests-- select { case <-cs.abort: @@ -1871,10 +2015,26 @@ func (cs *clientStream) awaitFlowControl(maxBytes int) (taken int32, err error) cs.flow.take(take) return take, nil } - cc.cond.Wait() + cc.condWait() } } +func validateHeaders(hdrs http.Header) string { + for k, vv := range hdrs { + if !httpguts.ValidHeaderFieldName(k) { + return fmt.Sprintf("name %q", k) + } + for _, v := range vv { + if !httpguts.ValidHeaderFieldValue(v) { + // Don't include the value in the error, + // because it may be sensitive. + return fmt.Sprintf("value for header %q", k) + } + } + } + return "" +} + var errNilRequestURL = errors.New("http2: Request.URI is nil") // requires cc.wmu be held. @@ -1912,19 +2072,14 @@ func (cc *ClientConn) encodeHeaders(req *http.Request, addGzipHeader bool, trail } } - // Check for any invalid headers and return an error before we + // Check for any invalid headers+trailers and return an error before we // potentially pollute our hpack state. (We want to be able to // continue to reuse the hpack encoder for future requests) - for k, vv := range req.Header { - if !httpguts.ValidHeaderFieldName(k) { - return nil, fmt.Errorf("invalid HTTP header name %q", k) - } - for _, v := range vv { - if !httpguts.ValidHeaderFieldValue(v) { - // Don't include the value in the error, because it may be sensitive. - return nil, fmt.Errorf("invalid HTTP header value for header %q", k) - } - } + if err := validateHeaders(req.Header); err != "" { + return nil, fmt.Errorf("invalid HTTP header %s", err) + } + if err := validateHeaders(req.Trailer); err != "" { + return nil, fmt.Errorf("invalid HTTP trailer %s", err) } enumerateHeaders := func(f func(name, value string)) { @@ -2143,7 +2298,7 @@ func (cc *ClientConn) forgetStreamID(id uint32) { } // Wake up writeRequestBody via clientStream.awaitFlowControl and // wake up RoundTrip if there is a pending request. - cc.cond.Broadcast() + cc.condBroadcast() closeOnIdle := cc.singleUse || cc.doNotReuse || cc.t.disableKeepAlives() || cc.goAway != nil if closeOnIdle && cc.streamsReserved == 0 && len(cc.streams) == 0 { @@ -2231,7 +2386,7 @@ func (rl *clientConnReadLoop) cleanup() { cs.abortStreamLocked(err) } } - cc.cond.Broadcast() + cc.condBroadcast() cc.mu.Unlock() } @@ -2266,10 +2421,9 @@ func (rl *clientConnReadLoop) run() error { cc := rl.cc gotSettings := false readIdleTimeout := cc.t.ReadIdleTimeout - var t *time.Timer + var t timer if readIdleTimeout != 0 { - t = time.AfterFunc(readIdleTimeout, cc.healthCheck) - defer t.Stop() + t = cc.afterFunc(readIdleTimeout, cc.healthCheck) } for { f, err := cc.fr.ReadFrame() @@ -2684,7 +2838,7 @@ func (rl *clientConnReadLoop) processData(f *DataFrame) error { }) return nil } - if !cs.firstByte { + if !cs.pastHeaders { cc.logf("protocol error: received DATA before a HEADERS frame") rl.endStreamError(cs, StreamError{ StreamID: f.StreamID, @@ -2867,7 +3021,7 @@ func (rl *clientConnReadLoop) processSettingsNoWrite(f *SettingsFrame) error { for _, cs := range cc.streams { cs.flow.add(delta) } - cc.cond.Broadcast() + cc.condBroadcast() cc.initialWindowSize = s.Val case SettingHeaderTableSize: @@ -2911,9 +3065,18 @@ func (rl *clientConnReadLoop) processWindowUpdate(f *WindowUpdateFrame) error { fl = &cs.flow } if !fl.add(int32(f.Increment)) { + // For stream, the sender sends RST_STREAM with an error code of FLOW_CONTROL_ERROR + if cs != nil { + rl.endStreamError(cs, StreamError{ + StreamID: f.StreamID, + Code: ErrCodeFlowControl, + }) + return nil + } + return ConnectionError(ErrCodeFlowControl) } - cc.cond.Broadcast() + cc.condBroadcast() return nil } @@ -2955,24 +3118,38 @@ func (cc *ClientConn) Ping(ctx context.Context) error { } cc.mu.Unlock() } - errc := make(chan error, 1) - go func() { + var pingError error + errc := make(chan struct{}) + cc.goRun(func() { cc.wmu.Lock() defer cc.wmu.Unlock() - if err := cc.fr.WritePing(false, p); err != nil { - errc <- err + if pingError = cc.fr.WritePing(false, p); pingError != nil { + close(errc) return } - if err := cc.bw.Flush(); err != nil { - errc <- err + if pingError = cc.bw.Flush(); pingError != nil { + close(errc) return } - }() + }) + if cc.syncHooks != nil { + cc.syncHooks.blockUntil(func() bool { + select { + case <-c: + case <-errc: + case <-ctx.Done(): + case <-cc.readerDone: + default: + return false + } + return true + }) + } select { case <-c: return nil - case err := <-errc: - return err + case <-errc: + return pingError case <-ctx.Done(): return ctx.Err() case <-cc.readerDone: @@ -3141,9 +3318,17 @@ func (rt noDialH2RoundTripper) RoundTrip(req *http.Request) (*http.Response, err } func (t *Transport) idleConnTimeout() time.Duration { + // to keep things backwards compatible, we use non-zero values of + // IdleConnTimeout, followed by using the IdleConnTimeout on the underlying + // http1 transport, followed by 0 + if t.IdleConnTimeout != 0 { + return t.IdleConnTimeout + } + if t.t1 != nil { return t.t1.IdleConnTimeout } + return 0 } @@ -3201,3 +3386,34 @@ func traceFirstResponseByte(trace *httptrace.ClientTrace) { trace.GotFirstResponseByte() } } + +func traceHasWroteHeaderField(trace *httptrace.ClientTrace) bool { + return trace != nil && trace.WroteHeaderField != nil +} + +func traceWroteHeaderField(trace *httptrace.ClientTrace, k, v string) { + if trace != nil && trace.WroteHeaderField != nil { + trace.WroteHeaderField(k, []string{v}) + } +} + +func traceGot1xxResponseFunc(trace *httptrace.ClientTrace) func(int, textproto.MIMEHeader) error { + if trace != nil { + return trace.Got1xxResponse + } + return nil +} + +// dialTLSWithContext uses tls.Dialer, added in Go 1.15, to open a TLS +// connection. +func (t *Transport) dialTLSWithContext(ctx context.Context, network, addr string, cfg *tls.Config) (*tls.Conn, error) { + dialer := &tls.Dialer{ + Config: cfg, + } + cn, err := dialer.DialContext(ctx, network, addr) + if err != nil { + return nil, err + } + tlsCn := cn.(*tls.Conn) // DialContext comment promises this will always succeed + return tlsCn, nil +} diff --git a/vendor/golang.org/x/net/idna/go118.go b/vendor/golang.org/x/net/idna/go118.go index c5c4338db..712f1ad83 100644 --- a/vendor/golang.org/x/net/idna/go118.go +++ b/vendor/golang.org/x/net/idna/go118.go @@ -5,7 +5,6 @@ // license that can be found in the LICENSE file. //go:build go1.18 -// +build go1.18 package idna diff --git a/vendor/golang.org/x/net/idna/idna10.0.0.go b/vendor/golang.org/x/net/idna/idna10.0.0.go index 64ccf85fe..7b3717884 100644 --- a/vendor/golang.org/x/net/idna/idna10.0.0.go +++ b/vendor/golang.org/x/net/idna/idna10.0.0.go @@ -5,7 +5,6 @@ // license that can be found in the LICENSE file. //go:build go1.10 -// +build go1.10 // Package idna implements IDNA2008 using the compatibility processing // defined by UTS (Unicode Technical Standard) #46, which defines a standard to diff --git a/vendor/golang.org/x/net/idna/idna9.0.0.go b/vendor/golang.org/x/net/idna/idna9.0.0.go index ee1698cef..cc6a892a4 100644 --- a/vendor/golang.org/x/net/idna/idna9.0.0.go +++ b/vendor/golang.org/x/net/idna/idna9.0.0.go @@ -5,7 +5,6 @@ // license that can be found in the LICENSE file. //go:build !go1.10 -// +build !go1.10 // Package idna implements IDNA2008 using the compatibility processing // defined by UTS (Unicode Technical Standard) #46, which defines a standard to diff --git a/vendor/golang.org/x/net/idna/pre_go118.go b/vendor/golang.org/x/net/idna/pre_go118.go index 3aaccab1c..40e74bb3d 100644 --- a/vendor/golang.org/x/net/idna/pre_go118.go +++ b/vendor/golang.org/x/net/idna/pre_go118.go @@ -5,7 +5,6 @@ // license that can be found in the LICENSE file. //go:build !go1.18 -// +build !go1.18 package idna diff --git a/vendor/golang.org/x/net/idna/tables10.0.0.go b/vendor/golang.org/x/net/idna/tables10.0.0.go index d1d62ef45..c6c2bf10a 100644 --- a/vendor/golang.org/x/net/idna/tables10.0.0.go +++ b/vendor/golang.org/x/net/idna/tables10.0.0.go @@ -1,7 +1,6 @@ // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. //go:build go1.10 && !go1.13 -// +build go1.10,!go1.13 package idna diff --git a/vendor/golang.org/x/net/idna/tables11.0.0.go b/vendor/golang.org/x/net/idna/tables11.0.0.go index 167efba71..76789393c 100644 --- a/vendor/golang.org/x/net/idna/tables11.0.0.go +++ b/vendor/golang.org/x/net/idna/tables11.0.0.go @@ -1,7 +1,6 @@ // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. //go:build go1.13 && !go1.14 -// +build go1.13,!go1.14 package idna diff --git a/vendor/golang.org/x/net/idna/tables12.0.0.go b/vendor/golang.org/x/net/idna/tables12.0.0.go index ab40f7bcc..0600cd2ae 100644 --- a/vendor/golang.org/x/net/idna/tables12.0.0.go +++ b/vendor/golang.org/x/net/idna/tables12.0.0.go @@ -1,7 +1,6 @@ // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. //go:build go1.14 && !go1.16 -// +build go1.14,!go1.16 package idna diff --git a/vendor/golang.org/x/net/idna/tables13.0.0.go b/vendor/golang.org/x/net/idna/tables13.0.0.go index 66701eadf..2fb768ef6 100644 --- a/vendor/golang.org/x/net/idna/tables13.0.0.go +++ b/vendor/golang.org/x/net/idna/tables13.0.0.go @@ -1,7 +1,6 @@ // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. //go:build go1.16 && !go1.21 -// +build go1.16,!go1.21 package idna diff --git a/vendor/golang.org/x/net/idna/tables15.0.0.go b/vendor/golang.org/x/net/idna/tables15.0.0.go index 40033778f..5ff05fe1a 100644 --- a/vendor/golang.org/x/net/idna/tables15.0.0.go +++ b/vendor/golang.org/x/net/idna/tables15.0.0.go @@ -1,7 +1,6 @@ // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. //go:build go1.21 -// +build go1.21 package idna diff --git a/vendor/golang.org/x/net/idna/tables9.0.0.go b/vendor/golang.org/x/net/idna/tables9.0.0.go index 4074b5332..0f25e84ca 100644 --- a/vendor/golang.org/x/net/idna/tables9.0.0.go +++ b/vendor/golang.org/x/net/idna/tables9.0.0.go @@ -1,7 +1,6 @@ // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. //go:build !go1.10 -// +build !go1.10 package idna diff --git a/vendor/golang.org/x/net/idna/trie12.0.0.go b/vendor/golang.org/x/net/idna/trie12.0.0.go index bb63f904b..8a75b9667 100644 --- a/vendor/golang.org/x/net/idna/trie12.0.0.go +++ b/vendor/golang.org/x/net/idna/trie12.0.0.go @@ -5,7 +5,6 @@ // license that can be found in the LICENSE file. //go:build !go1.16 -// +build !go1.16 package idna diff --git a/vendor/golang.org/x/net/idna/trie13.0.0.go b/vendor/golang.org/x/net/idna/trie13.0.0.go index 7d68a8dc1..fa45bb907 100644 --- a/vendor/golang.org/x/net/idna/trie13.0.0.go +++ b/vendor/golang.org/x/net/idna/trie13.0.0.go @@ -5,7 +5,6 @@ // license that can be found in the LICENSE file. //go:build go1.16 -// +build go1.16 package idna diff --git a/vendor/golang.org/x/sys/unix/aliases.go b/vendor/golang.org/x/sys/unix/aliases.go index e7d3df4bd..b0e419857 100644 --- a/vendor/golang.org/x/sys/unix/aliases.go +++ b/vendor/golang.org/x/sys/unix/aliases.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build (aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris || zos) && go1.9 +//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris || zos package unix diff --git a/vendor/golang.org/x/sys/unix/syscall_darwin_libSystem.go b/vendor/golang.org/x/sys/unix/syscall_darwin_libSystem.go index 16dc69937..2f0fa76e4 100644 --- a/vendor/golang.org/x/sys/unix/syscall_darwin_libSystem.go +++ b/vendor/golang.org/x/sys/unix/syscall_darwin_libSystem.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build darwin && go1.12 +//go:build darwin package unix diff --git a/vendor/golang.org/x/sys/unix/syscall_freebsd.go b/vendor/golang.org/x/sys/unix/syscall_freebsd.go index 64d1bb4db..2b57e0f73 100644 --- a/vendor/golang.org/x/sys/unix/syscall_freebsd.go +++ b/vendor/golang.org/x/sys/unix/syscall_freebsd.go @@ -13,6 +13,7 @@ package unix import ( + "errors" "sync" "unsafe" ) @@ -169,25 +170,26 @@ func Getfsstat(buf []Statfs_t, flags int) (n int, err error) { func Uname(uname *Utsname) error { mib := []_C_int{CTL_KERN, KERN_OSTYPE} n := unsafe.Sizeof(uname.Sysname) - if err := sysctl(mib, &uname.Sysname[0], &n, nil, 0); err != nil { + // Suppress ENOMEM errors to be compatible with the C library __xuname() implementation. + if err := sysctl(mib, &uname.Sysname[0], &n, nil, 0); err != nil && !errors.Is(err, ENOMEM) { return err } mib = []_C_int{CTL_KERN, KERN_HOSTNAME} n = unsafe.Sizeof(uname.Nodename) - if err := sysctl(mib, &uname.Nodename[0], &n, nil, 0); err != nil { + if err := sysctl(mib, &uname.Nodename[0], &n, nil, 0); err != nil && !errors.Is(err, ENOMEM) { return err } mib = []_C_int{CTL_KERN, KERN_OSRELEASE} n = unsafe.Sizeof(uname.Release) - if err := sysctl(mib, &uname.Release[0], &n, nil, 0); err != nil { + if err := sysctl(mib, &uname.Release[0], &n, nil, 0); err != nil && !errors.Is(err, ENOMEM) { return err } mib = []_C_int{CTL_KERN, KERN_VERSION} n = unsafe.Sizeof(uname.Version) - if err := sysctl(mib, &uname.Version[0], &n, nil, 0); err != nil { + if err := sysctl(mib, &uname.Version[0], &n, nil, 0); err != nil && !errors.Is(err, ENOMEM) { return err } @@ -205,7 +207,7 @@ func Uname(uname *Utsname) error { mib = []_C_int{CTL_HW, HW_MACHINE} n = unsafe.Sizeof(uname.Machine) - if err := sysctl(mib, &uname.Machine[0], &n, nil, 0); err != nil { + if err := sysctl(mib, &uname.Machine[0], &n, nil, 0); err != nil && !errors.Is(err, ENOMEM) { return err } diff --git a/vendor/golang.org/x/sys/unix/syscall_linux.go b/vendor/golang.org/x/sys/unix/syscall_linux.go index 0f85e29e6..5682e2628 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux.go @@ -1849,6 +1849,105 @@ func Dup2(oldfd, newfd int) error { //sys Fsmount(fd int, flags int, mountAttrs int) (fsfd int, err error) //sys Fsopen(fsName string, flags int) (fd int, err error) //sys Fspick(dirfd int, pathName string, flags int) (fd int, err error) + +//sys fsconfig(fd int, cmd uint, key *byte, value *byte, aux int) (err error) + +func fsconfigCommon(fd int, cmd uint, key string, value *byte, aux int) (err error) { + var keyp *byte + if keyp, err = BytePtrFromString(key); err != nil { + return + } + return fsconfig(fd, cmd, keyp, value, aux) +} + +// FsconfigSetFlag is equivalent to fsconfig(2) called +// with cmd == FSCONFIG_SET_FLAG. +// +// fd is the filesystem context to act upon. +// key the parameter key to set. +func FsconfigSetFlag(fd int, key string) (err error) { + return fsconfigCommon(fd, FSCONFIG_SET_FLAG, key, nil, 0) +} + +// FsconfigSetString is equivalent to fsconfig(2) called +// with cmd == FSCONFIG_SET_STRING. +// +// fd is the filesystem context to act upon. +// key the parameter key to set. +// value is the parameter value to set. +func FsconfigSetString(fd int, key string, value string) (err error) { + var valuep *byte + if valuep, err = BytePtrFromString(value); err != nil { + return + } + return fsconfigCommon(fd, FSCONFIG_SET_STRING, key, valuep, 0) +} + +// FsconfigSetBinary is equivalent to fsconfig(2) called +// with cmd == FSCONFIG_SET_BINARY. +// +// fd is the filesystem context to act upon. +// key the parameter key to set. +// value is the parameter value to set. +func FsconfigSetBinary(fd int, key string, value []byte) (err error) { + if len(value) == 0 { + return EINVAL + } + return fsconfigCommon(fd, FSCONFIG_SET_BINARY, key, &value[0], len(value)) +} + +// FsconfigSetPath is equivalent to fsconfig(2) called +// with cmd == FSCONFIG_SET_PATH. +// +// fd is the filesystem context to act upon. +// key the parameter key to set. +// path is a non-empty path for specified key. +// atfd is a file descriptor at which to start lookup from or AT_FDCWD. +func FsconfigSetPath(fd int, key string, path string, atfd int) (err error) { + var valuep *byte + if valuep, err = BytePtrFromString(path); err != nil { + return + } + return fsconfigCommon(fd, FSCONFIG_SET_PATH, key, valuep, atfd) +} + +// FsconfigSetPathEmpty is equivalent to fsconfig(2) called +// with cmd == FSCONFIG_SET_PATH_EMPTY. The same as +// FconfigSetPath but with AT_PATH_EMPTY implied. +func FsconfigSetPathEmpty(fd int, key string, path string, atfd int) (err error) { + var valuep *byte + if valuep, err = BytePtrFromString(path); err != nil { + return + } + return fsconfigCommon(fd, FSCONFIG_SET_PATH_EMPTY, key, valuep, atfd) +} + +// FsconfigSetFd is equivalent to fsconfig(2) called +// with cmd == FSCONFIG_SET_FD. +// +// fd is the filesystem context to act upon. +// key the parameter key to set. +// value is a file descriptor to be assigned to specified key. +func FsconfigSetFd(fd int, key string, value int) (err error) { + return fsconfigCommon(fd, FSCONFIG_SET_FD, key, nil, value) +} + +// FsconfigCreate is equivalent to fsconfig(2) called +// with cmd == FSCONFIG_CMD_CREATE. +// +// fd is the filesystem context to act upon. +func FsconfigCreate(fd int) (err error) { + return fsconfig(fd, FSCONFIG_CMD_CREATE, nil, nil, 0) +} + +// FsconfigReconfigure is equivalent to fsconfig(2) called +// with cmd == FSCONFIG_CMD_RECONFIGURE. +// +// fd is the filesystem context to act upon. +func FsconfigReconfigure(fd int) (err error) { + return fsconfig(fd, FSCONFIG_CMD_RECONFIGURE, nil, nil, 0) +} + //sys Getdents(fd int, buf []byte) (n int, err error) = SYS_GETDENTS64 //sysnb Getpgid(pid int) (pgid int, err error) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux.go b/vendor/golang.org/x/sys/unix/zsyscall_linux.go index 1488d2712..87d8612a1 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux.go @@ -906,6 +906,16 @@ func Fspick(dirfd int, pathName string, flags int) (fd int, err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func fsconfig(fd int, cmd uint, key *byte, value *byte, aux int) (err error) { + _, _, e1 := Syscall6(SYS_FSCONFIG, uintptr(fd), uintptr(cmd), uintptr(unsafe.Pointer(key)), uintptr(unsafe.Pointer(value)), uintptr(aux), 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Getdents(fd int, buf []byte) (n int, err error) { var _p0 unsafe.Pointer if len(buf) > 0 { diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux.go b/vendor/golang.org/x/sys/unix/ztypes_linux.go index dc0c955ee..eff6bcdef 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux.go @@ -836,6 +836,15 @@ const ( FSPICK_EMPTY_PATH = 0x8 FSMOUNT_CLOEXEC = 0x1 + + FSCONFIG_SET_FLAG = 0x0 + FSCONFIG_SET_STRING = 0x1 + FSCONFIG_SET_BINARY = 0x2 + FSCONFIG_SET_PATH = 0x3 + FSCONFIG_SET_PATH_EMPTY = 0x4 + FSCONFIG_SET_FD = 0x5 + FSCONFIG_CMD_CREATE = 0x6 + FSCONFIG_CMD_RECONFIGURE = 0x7 ) type OpenHow struct { @@ -1550,6 +1559,7 @@ const ( IFLA_DEVLINK_PORT = 0x3e IFLA_GSO_IPV4_MAX_SIZE = 0x3f IFLA_GRO_IPV4_MAX_SIZE = 0x40 + IFLA_DPLL_PIN = 0x41 IFLA_PROTO_DOWN_REASON_UNSPEC = 0x0 IFLA_PROTO_DOWN_REASON_MASK = 0x1 IFLA_PROTO_DOWN_REASON_VALUE = 0x2 @@ -1565,6 +1575,7 @@ const ( IFLA_INET6_ICMP6STATS = 0x6 IFLA_INET6_TOKEN = 0x7 IFLA_INET6_ADDR_GEN_MODE = 0x8 + IFLA_INET6_RA_MTU = 0x9 IFLA_BR_UNSPEC = 0x0 IFLA_BR_FORWARD_DELAY = 0x1 IFLA_BR_HELLO_TIME = 0x2 @@ -1612,6 +1623,9 @@ const ( IFLA_BR_MCAST_MLD_VERSION = 0x2c IFLA_BR_VLAN_STATS_PER_PORT = 0x2d IFLA_BR_MULTI_BOOLOPT = 0x2e + IFLA_BR_MCAST_QUERIER_STATE = 0x2f + IFLA_BR_FDB_N_LEARNED = 0x30 + IFLA_BR_FDB_MAX_LEARNED = 0x31 IFLA_BRPORT_UNSPEC = 0x0 IFLA_BRPORT_STATE = 0x1 IFLA_BRPORT_PRIORITY = 0x2 @@ -1649,6 +1663,14 @@ const ( IFLA_BRPORT_BACKUP_PORT = 0x22 IFLA_BRPORT_MRP_RING_OPEN = 0x23 IFLA_BRPORT_MRP_IN_OPEN = 0x24 + IFLA_BRPORT_MCAST_EHT_HOSTS_LIMIT = 0x25 + IFLA_BRPORT_MCAST_EHT_HOSTS_CNT = 0x26 + IFLA_BRPORT_LOCKED = 0x27 + IFLA_BRPORT_MAB = 0x28 + IFLA_BRPORT_MCAST_N_GROUPS = 0x29 + IFLA_BRPORT_MCAST_MAX_GROUPS = 0x2a + IFLA_BRPORT_NEIGH_VLAN_SUPPRESS = 0x2b + IFLA_BRPORT_BACKUP_NHID = 0x2c IFLA_INFO_UNSPEC = 0x0 IFLA_INFO_KIND = 0x1 IFLA_INFO_DATA = 0x2 @@ -1670,6 +1692,9 @@ const ( IFLA_MACVLAN_MACADDR = 0x4 IFLA_MACVLAN_MACADDR_DATA = 0x5 IFLA_MACVLAN_MACADDR_COUNT = 0x6 + IFLA_MACVLAN_BC_QUEUE_LEN = 0x7 + IFLA_MACVLAN_BC_QUEUE_LEN_USED = 0x8 + IFLA_MACVLAN_BC_CUTOFF = 0x9 IFLA_VRF_UNSPEC = 0x0 IFLA_VRF_TABLE = 0x1 IFLA_VRF_PORT_UNSPEC = 0x0 @@ -1693,9 +1718,22 @@ const ( IFLA_XFRM_UNSPEC = 0x0 IFLA_XFRM_LINK = 0x1 IFLA_XFRM_IF_ID = 0x2 + IFLA_XFRM_COLLECT_METADATA = 0x3 IFLA_IPVLAN_UNSPEC = 0x0 IFLA_IPVLAN_MODE = 0x1 IFLA_IPVLAN_FLAGS = 0x2 + NETKIT_NEXT = -0x1 + NETKIT_PASS = 0x0 + NETKIT_DROP = 0x2 + NETKIT_REDIRECT = 0x7 + NETKIT_L2 = 0x0 + NETKIT_L3 = 0x1 + IFLA_NETKIT_UNSPEC = 0x0 + IFLA_NETKIT_PEER_INFO = 0x1 + IFLA_NETKIT_PRIMARY = 0x2 + IFLA_NETKIT_POLICY = 0x3 + IFLA_NETKIT_PEER_POLICY = 0x4 + IFLA_NETKIT_MODE = 0x5 IFLA_VXLAN_UNSPEC = 0x0 IFLA_VXLAN_ID = 0x1 IFLA_VXLAN_GROUP = 0x2 @@ -1726,6 +1764,8 @@ const ( IFLA_VXLAN_GPE = 0x1b IFLA_VXLAN_TTL_INHERIT = 0x1c IFLA_VXLAN_DF = 0x1d + IFLA_VXLAN_VNIFILTER = 0x1e + IFLA_VXLAN_LOCALBYPASS = 0x1f IFLA_GENEVE_UNSPEC = 0x0 IFLA_GENEVE_ID = 0x1 IFLA_GENEVE_REMOTE = 0x2 @@ -1740,6 +1780,7 @@ const ( IFLA_GENEVE_LABEL = 0xb IFLA_GENEVE_TTL_INHERIT = 0xc IFLA_GENEVE_DF = 0xd + IFLA_GENEVE_INNER_PROTO_INHERIT = 0xe IFLA_BAREUDP_UNSPEC = 0x0 IFLA_BAREUDP_PORT = 0x1 IFLA_BAREUDP_ETHERTYPE = 0x2 @@ -1752,6 +1793,8 @@ const ( IFLA_GTP_FD1 = 0x2 IFLA_GTP_PDP_HASHSIZE = 0x3 IFLA_GTP_ROLE = 0x4 + IFLA_GTP_CREATE_SOCKETS = 0x5 + IFLA_GTP_RESTART_COUNT = 0x6 IFLA_BOND_UNSPEC = 0x0 IFLA_BOND_MODE = 0x1 IFLA_BOND_ACTIVE_SLAVE = 0x2 @@ -1781,6 +1824,9 @@ const ( IFLA_BOND_AD_ACTOR_SYSTEM = 0x1a IFLA_BOND_TLB_DYNAMIC_LB = 0x1b IFLA_BOND_PEER_NOTIF_DELAY = 0x1c + IFLA_BOND_AD_LACP_ACTIVE = 0x1d + IFLA_BOND_MISSED_MAX = 0x1e + IFLA_BOND_NS_IP6_TARGET = 0x1f IFLA_BOND_AD_INFO_UNSPEC = 0x0 IFLA_BOND_AD_INFO_AGGREGATOR = 0x1 IFLA_BOND_AD_INFO_NUM_PORTS = 0x2 @@ -1796,6 +1842,7 @@ const ( IFLA_BOND_SLAVE_AD_AGGREGATOR_ID = 0x6 IFLA_BOND_SLAVE_AD_ACTOR_OPER_PORT_STATE = 0x7 IFLA_BOND_SLAVE_AD_PARTNER_OPER_PORT_STATE = 0x8 + IFLA_BOND_SLAVE_PRIO = 0x9 IFLA_VF_INFO_UNSPEC = 0x0 IFLA_VF_INFO = 0x1 IFLA_VF_UNSPEC = 0x0 @@ -1854,8 +1901,16 @@ const ( IFLA_STATS_LINK_XSTATS_SLAVE = 0x3 IFLA_STATS_LINK_OFFLOAD_XSTATS = 0x4 IFLA_STATS_AF_SPEC = 0x5 + IFLA_STATS_GETSET_UNSPEC = 0x0 + IFLA_STATS_GET_FILTERS = 0x1 + IFLA_STATS_SET_OFFLOAD_XSTATS_L3_STATS = 0x2 IFLA_OFFLOAD_XSTATS_UNSPEC = 0x0 IFLA_OFFLOAD_XSTATS_CPU_HIT = 0x1 + IFLA_OFFLOAD_XSTATS_HW_S_INFO = 0x2 + IFLA_OFFLOAD_XSTATS_L3_STATS = 0x3 + IFLA_OFFLOAD_XSTATS_HW_S_INFO_UNSPEC = 0x0 + IFLA_OFFLOAD_XSTATS_HW_S_INFO_REQUEST = 0x1 + IFLA_OFFLOAD_XSTATS_HW_S_INFO_USED = 0x2 IFLA_XDP_UNSPEC = 0x0 IFLA_XDP_FD = 0x1 IFLA_XDP_ATTACHED = 0x2 @@ -1885,6 +1940,11 @@ const ( IFLA_RMNET_UNSPEC = 0x0 IFLA_RMNET_MUX_ID = 0x1 IFLA_RMNET_FLAGS = 0x2 + IFLA_MCTP_UNSPEC = 0x0 + IFLA_MCTP_NET = 0x1 + IFLA_DSA_UNSPEC = 0x0 + IFLA_DSA_CONDUIT = 0x1 + IFLA_DSA_MASTER = 0x1 ) const ( diff --git a/vendor/modules.txt b/vendor/modules.txt index 7fee36ffa..47e092460 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -503,7 +503,7 @@ go.uber.org/zap/zapcore # go4.org v0.0.0-20200104003542-c7e774b10ea0 ## explicit go4.org/errorutil -# golang.org/x/crypto v0.17.0 +# golang.org/x/crypto v0.21.0 ## explicit; go 1.18 golang.org/x/crypto/bcrypt golang.org/x/crypto/blowfish @@ -517,8 +517,8 @@ golang.org/x/exp/maps golang.org/x/mod/internal/lazyregexp golang.org/x/mod/module golang.org/x/mod/semver -# golang.org/x/net v0.17.0 -## explicit; go 1.17 +# golang.org/x/net v0.23.0 +## explicit; go 1.18 golang.org/x/net/context golang.org/x/net/html golang.org/x/net/html/atom @@ -538,13 +538,13 @@ golang.org/x/oauth2/internal # golang.org/x/sync v0.4.0 ## explicit; go 1.17 golang.org/x/sync/errgroup -# golang.org/x/sys v0.17.0 +# golang.org/x/sys v0.18.0 ## explicit; go 1.18 golang.org/x/sys/execabs golang.org/x/sys/plan9 golang.org/x/sys/unix golang.org/x/sys/windows -# golang.org/x/term v0.15.0 +# golang.org/x/term v0.18.0 ## explicit; go 1.18 golang.org/x/term # golang.org/x/text v0.14.0 From 6ac959b4b1698e2c7c5baa9688a758aff2c676a0 Mon Sep 17 00:00:00 2001 From: Sebastian Sch Date: Wed, 1 May 2024 13:02:54 +0300 Subject: [PATCH 20/21] don't reset the device plugin twice This change is to fix a regression introduce with the nic check. We can have a race condition to will reset the device plugin when it's not needed. This can cause pods trying to start to get into a kubelet error pointing that the device plugin was not available Signed-off-by: Sebastian Sch --- pkg/daemon/daemon.go | 114 +++++++++++++++----------- pkg/plugins/generic/generic_plugin.go | 5 ++ 2 files changed, 69 insertions(+), 50 deletions(-) diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index 950287a16..0fd1ec2cb 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -322,51 +322,58 @@ func (dn *Daemon) nodeStateSyncHandler() error { } } - if vars.UsingSystemdMode && dn.currentNodeState.GetGeneration() == latest && !dn.isDrainCompleted() { - serviceEnabled, err := dn.HostHelpers.IsServiceEnabled(systemd.SriovServicePath) - if err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): failed to check if sriov-config service exist on host") - return err - } - postNetworkServiceEnabled, err := dn.HostHelpers.IsServiceEnabled(systemd.SriovPostNetworkServicePath) - if err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): failed to check if sriov-config-post-network service exist on host") - return err - } - - // if the service doesn't exist we should continue to let the k8s plugin to create the service files - // this is only for k8s base environments, for openshift the sriov-operator creates a machine config to will apply - // the system service and reboot the node the config-daemon doesn't need to do anything. - if !(serviceEnabled && postNetworkServiceEnabled) { - sriovResult = &systemd.SriovResult{SyncStatus: consts.SyncStatusFailed, - LastSyncError: fmt.Sprintf("some sriov systemd services are not available on node: "+ - "sriov-config available:%t, sriov-config-post-network available:%t", serviceEnabled, postNetworkServiceEnabled)} - } else { - sriovResult, err = systemd.ReadSriovResult() + skipReconciliation := true + // if the operator complete the drain operator we should continue the configuration + if !dn.isDrainCompleted() { + if vars.UsingSystemdMode && dn.currentNodeState.GetGeneration() == latest { + serviceEnabled, err := dn.HostHelpers.IsServiceEnabled(systemd.SriovServicePath) if err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): failed to load sriov result file from host") + log.Log.Error(err, "nodeStateSyncHandler(): failed to check if sriov-config service exist on host") + return err + } + postNetworkServiceEnabled, err := dn.HostHelpers.IsServiceEnabled(systemd.SriovPostNetworkServicePath) + if err != nil { + log.Log.Error(err, "nodeStateSyncHandler(): failed to check if sriov-config-post-network service exist on host") return err } - } - if sriovResult.LastSyncError != "" || sriovResult.SyncStatus == consts.SyncStatusFailed { - log.Log.Info("nodeStateSyncHandler(): sync failed systemd service error", "last-sync-error", sriovResult.LastSyncError) - // add the error but don't requeue - dn.refreshCh <- Message{ - syncStatus: consts.SyncStatusFailed, - lastSyncError: sriovResult.LastSyncError, + // if the service doesn't exist we should continue to let the k8s plugin to create the service files + // this is only for k8s base environments, for openshift the sriov-operator creates a machine config to will apply + // the system service and reboot the node the config-daemon doesn't need to do anything. + if !(serviceEnabled && postNetworkServiceEnabled) { + sriovResult = &systemd.SriovResult{SyncStatus: consts.SyncStatusFailed, + LastSyncError: fmt.Sprintf("some sriov systemd services are not available on node: "+ + "sriov-config available:%t, sriov-config-post-network available:%t", serviceEnabled, postNetworkServiceEnabled)} + } else { + sriovResult, err = systemd.ReadSriovResult() + if err != nil { + log.Log.Error(err, "nodeStateSyncHandler(): failed to load sriov result file from host") + return err + } + } + if sriovResult.LastSyncError != "" || sriovResult.SyncStatus == consts.SyncStatusFailed { + log.Log.Info("nodeStateSyncHandler(): sync failed systemd service error", "last-sync-error", sriovResult.LastSyncError) + + // add the error but don't requeue + dn.refreshCh <- Message{ + syncStatus: consts.SyncStatusFailed, + lastSyncError: sriovResult.LastSyncError, + } + <-dn.syncCh + return nil } - <-dn.syncCh - return nil } - } - skip, err := dn.shouldSkipReconciliation(dn.desiredNodeState) - if err != nil { - return err + skipReconciliation, err = dn.shouldSkipReconciliation(dn.desiredNodeState) + if err != nil { + return err + } } - // Reconcile only when there are changes in the spec or status of managed VFs. - if skip { + + // we are done with the configuration just return here + if dn.currentNodeState.GetGeneration() == dn.desiredNodeState.GetGeneration() && + dn.desiredNodeState.Status.SyncStatus == consts.SyncStatusSucceeded && skipReconciliation { + log.Log.Info("Current state and desire state are equal together with sync status succeeded nothing to do") return nil } @@ -443,17 +450,6 @@ func (dn *Daemon) nodeStateSyncHandler() error { log.Log.V(0).Info("nodeStateSyncHandler(): aggregated daemon", "drain-required", reqDrain, "reboot-required", reqReboot, "disable-drain", dn.disableDrain) - for k, p := range dn.loadedPlugins { - // Skip both the general and virtual plugin apply them last - if k != GenericPluginName && k != VirtualPluginName { - err := p.Apply() - if err != nil { - log.Log.Error(err, "nodeStateSyncHandler(): plugin Apply failed", "plugin-name", k) - return err - } - } - } - // handle drain only if the plugin request drain, or we are already in a draining request state if reqDrain || !utils.ObjectHasAnnotation(dn.desiredNodeState, consts.NodeStateDrainAnnotationCurrent, @@ -468,6 +464,20 @@ func (dn *Daemon) nodeStateSyncHandler() error { } } + // apply the vendor plugins after we are done with drain if needed + for k, p := range dn.loadedPlugins { + // Skip both the general and virtual plugin apply them last + if k != GenericPluginName && k != VirtualPluginName { + err := p.Apply() + if err != nil { + log.Log.Error(err, "nodeStateSyncHandler(): plugin Apply failed", "plugin-name", k) + return err + } + } + } + + // if we don't need to reboot, or we are not doing the configuration in systemd + // we apply the generic plugin if !reqReboot && !vars.UsingSystemdMode { // For BareMetal machines apply the generic plugin selectedPlugin, ok := dn.loadedPlugins[GenericPluginName] @@ -539,6 +549,7 @@ func (dn *Daemon) nodeStateSyncHandler() error { } func (dn *Daemon) shouldSkipReconciliation(latestState *sriovnetworkv1.SriovNetworkNodeState) (bool, error) { + log.Log.V(0).Info("shouldSkipReconciliation()") var err error // Skip when SriovNetworkNodeState object has just been created. @@ -563,15 +574,18 @@ func (dn *Daemon) shouldSkipReconciliation(latestState *sriovnetworkv1.SriovNetw return true, nil } - // Verify changes in the spec of the SriovNetworkNodeState CR. - if dn.currentNodeState.GetGeneration() == latestState.GetGeneration() && !dn.isDrainCompleted() { + // Verify changes in the status of the SriovNetworkNodeState CR. + if dn.currentNodeState.GetGeneration() == latestState.GetGeneration() { + log.Log.V(0).Info("shouldSkipReconciliation() verifying status change") for _, p := range dn.loadedPlugins { // Verify changes in the status of the SriovNetworkNodeState CR. + log.Log.V(0).Info("shouldSkipReconciliation(): verifying status change for plugin", "pluginName", p.Name()) changed, err := p.CheckStatusChanges(latestState) if err != nil { return false, err } if changed { + log.Log.V(0).Info("shouldSkipReconciliation(): plugin require change", "pluginName", p.Name()) return false, nil } } diff --git a/pkg/plugins/generic/generic_plugin.go b/pkg/plugins/generic/generic_plugin.go index 5e0827c69..b01653d90 100644 --- a/pkg/plugins/generic/generic_plugin.go +++ b/pkg/plugins/generic/generic_plugin.go @@ -167,6 +167,11 @@ func (p *GenericPlugin) CheckStatusChanges(current *sriovnetworkv1.SriovNetworkN return false, err } + if len(missingKernelArgs) != 0 { + log.Log.V(0).Info("generic-plugin CheckStatusChanges(): kernel args missing", + "kernelArgs", missingKernelArgs) + } + return len(missingKernelArgs) != 0, nil } From 852e2b775015249ebc5bc71f7ab5e13e28354572 Mon Sep 17 00:00:00 2001 From: Sebastian Scheinkman Date: Tue, 7 May 2024 23:50:53 +0000 Subject: [PATCH 21/21] d/s run make bundle --- ...etwork-operator.clusterserviceversion.yaml | 2 +- ...vnetwork.openshift.io_sriovibnetworks.yaml | 29 ++++--- ...openshift.io_sriovnetworknodepolicies.yaml | 19 +++-- ...k.openshift.io_sriovnetworknodestates.yaml | 19 +++-- ....openshift.io_sriovnetworkpoolconfigs.yaml | 75 ++++++++++-------- ...iovnetwork.openshift.io_sriovnetworks.yaml | 41 +++++----- ...ork.openshift.io_sriovoperatorconfigs.yaml | 25 +++--- ...vnetwork.openshift.io_sriovibnetworks.yaml | 30 +++++--- ...openshift.io_sriovnetworknodepolicies.yaml | 20 +++-- ...k.openshift.io_sriovnetworknodestates.yaml | 20 +++-- ....openshift.io_sriovnetworkpoolconfigs.yaml | 76 ++++++++++--------- ...iovnetwork.openshift.io_sriovnetworks.yaml | 42 +++++----- ...ork.openshift.io_sriovoperatorconfigs.yaml | 26 ++++--- ...vnetwork.openshift.io_sriovibnetworks.yaml | 30 +++++--- ...openshift.io_sriovnetworknodepolicies.yaml | 20 +++-- ...k.openshift.io_sriovnetworknodestates.yaml | 20 +++-- ....openshift.io_sriovnetworkpoolconfigs.yaml | 76 ++++++++++--------- ...iovnetwork.openshift.io_sriovnetworks.yaml | 42 +++++----- ...ork.openshift.io_sriovoperatorconfigs.yaml | 26 ++++--- ...etwork-operator.clusterserviceversion.yaml | 2 +- ...vnetwork.openshift.io_sriovibnetworks.yaml | 29 ++++--- ...openshift.io_sriovnetworknodepolicies.yaml | 19 +++-- ...k.openshift.io_sriovnetworknodestates.yaml | 19 +++-- ....openshift.io_sriovnetworkpoolconfigs.yaml | 75 ++++++++++-------- ...iovnetwork.openshift.io_sriovnetworks.yaml | 41 +++++----- ...ork.openshift.io_sriovoperatorconfigs.yaml | 25 +++--- 26 files changed, 494 insertions(+), 354 deletions(-) diff --git a/bundle/manifests/sriov-network-operator.clusterserviceversion.yaml b/bundle/manifests/sriov-network-operator.clusterserviceversion.yaml index bbdff4d5b..fa5880465 100644 --- a/bundle/manifests/sriov-network-operator.clusterserviceversion.yaml +++ b/bundle/manifests/sriov-network-operator.clusterserviceversion.yaml @@ -100,7 +100,7 @@ metadata: categories: Networking certified: "false" containerImage: quay.io/openshift/origin-sriov-network-operator:4.16 - createdAt: "2024-04-18T13:41:29Z" + createdAt: "2024-05-07T23:50:52Z" description: An operator for configuring SR-IOV components and initializing SRIOV network devices in Openshift cluster. features.operators.openshift.io/cnf: "false" diff --git a/bundle/manifests/sriovnetwork.openshift.io_sriovibnetworks.yaml b/bundle/manifests/sriovnetwork.openshift.io_sriovibnetworks.yaml index fa5ea4c75..c8eb80e6b 100644 --- a/bundle/manifests/sriovnetwork.openshift.io_sriovibnetworks.yaml +++ b/bundle/manifests/sriovnetwork.openshift.io_sriovibnetworks.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.14.0 creationTimestamp: null name: sriovibnetworks.sriovnetwork.openshift.io spec: @@ -20,14 +20,19 @@ spec: description: SriovIBNetwork is the Schema for the sriovibnetworks API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -35,8 +40,9 @@ spec: description: SriovIBNetworkSpec defines the desired state of SriovIBNetwork properties: capabilities: - description: 'Capabilities to be configured for this network. Capabilities - supported: (infinibandGUID), e.g. ''{"infinibandGUID": true}''' + description: |- + Capabilities to be configured for this network. + Capabilities supported: (infinibandGUID), e.g. '{"infinibandGUID": true}' type: string ipam: description: IPAM configuration to be used for this network. @@ -49,8 +55,9 @@ spec: - disable type: string metaPlugins: - description: MetaPluginsConfig configuration to be used in order to - chain metaplugins to the sriov interface returned by the operator. + description: |- + MetaPluginsConfig configuration to be used in order to chain metaplugins to the sriov interface returned + by the operator. type: string networkNamespace: description: Namespace of the NetworkAttachmentDefinition custom resource diff --git a/bundle/manifests/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml b/bundle/manifests/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml index 2de8005b8..e3af6cc47 100644 --- a/bundle/manifests/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml +++ b/bundle/manifests/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.14.0 creationTimestamp: null name: sriovnetworknodepolicies.sriovnetwork.openshift.io spec: @@ -21,14 +21,19 @@ spec: API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object diff --git a/bundle/manifests/sriovnetwork.openshift.io_sriovnetworknodestates.yaml b/bundle/manifests/sriovnetwork.openshift.io_sriovnetworknodestates.yaml index db2990a3c..55cdde2ed 100644 --- a/bundle/manifests/sriovnetwork.openshift.io_sriovnetworknodestates.yaml +++ b/bundle/manifests/sriovnetwork.openshift.io_sriovnetworknodestates.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.14.0 creationTimestamp: null name: sriovnetworknodestates.sriovnetwork.openshift.io spec: @@ -34,14 +34,19 @@ spec: API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object diff --git a/bundle/manifests/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml b/bundle/manifests/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml index a66384769..49f9cf60d 100644 --- a/bundle/manifests/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml +++ b/bundle/manifests/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.14.0 creationTimestamp: null name: sriovnetworkpoolconfigs.sriovnetwork.openshift.io spec: @@ -21,14 +21,19 @@ spec: API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -39,13 +44,15 @@ spec: anyOf: - type: integer - type: string - description: "maxUnavailable defines either an integer number or percentage - of nodes in the pool that can go Unavailable during an update. \n - A value larger than 1 will mean multiple nodes going unavailable - during the update, which may affect your workload stress on the - remaining nodes. Drain will respect Pod Disruption Budgets (PDBs) - such as etcd quorum guards, even if maxUnavailable is greater than - one." + description: |- + maxUnavailable defines either an integer number or percentage + of nodes in the pool that can go Unavailable during an update. + + + A value larger than 1 will mean multiple nodes going unavailable during + the update, which may affect your workload stress on the remaining nodes. + Drain will respect Pod Disruption Budgets (PDBs) such as etcd quorum guards, + even if maxUnavailable is greater than one. x-kubernetes-int-or-string: true nodeSelector: description: nodeSelector specifies a label selector for Nodes @@ -54,24 +61,24 @@ spec: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the key - and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to - a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a strategic + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic merge patch. items: type: string @@ -84,22 +91,24 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object + x-kubernetes-map-type: atomic ovsHardwareOffloadConfig: description: OvsHardwareOffloadConfig describes the OVS HWOL configuration for selected Nodes properties: name: - description: 'Name is mandatory and must be unique. On Kubernetes: - Name is the name of OvsHardwareOffloadConfig On OpenShift: Name - is the name of MachineConfigPool to be enabled with OVS hardware - offload' + description: |- + Name is mandatory and must be unique. + On Kubernetes: + Name is the name of OvsHardwareOffloadConfig + On OpenShift: + Name is the name of MachineConfigPool to be enabled with OVS hardware offload type: string type: object type: object diff --git a/bundle/manifests/sriovnetwork.openshift.io_sriovnetworks.yaml b/bundle/manifests/sriovnetwork.openshift.io_sriovnetworks.yaml index 0c1034923..067e53a7f 100644 --- a/bundle/manifests/sriovnetwork.openshift.io_sriovnetworks.yaml +++ b/bundle/manifests/sriovnetwork.openshift.io_sriovnetworks.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.14.0 creationTimestamp: null name: sriovnetworks.sriovnetwork.openshift.io spec: @@ -20,14 +20,19 @@ spec: description: SriovNetwork is the Schema for the sriovnetworks API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -35,8 +40,9 @@ spec: description: SriovNetworkSpec defines the desired state of SriovNetwork properties: capabilities: - description: 'Capabilities to be configured for this network. Capabilities - supported: (mac|ips), e.g. ''{"mac": true}''' + description: |- + Capabilities to be configured for this network. + Capabilities supported: (mac|ips), e.g. '{"mac": true}' type: string ipam: description: IPAM configuration to be used for this network. @@ -49,15 +55,15 @@ spec: - disable type: string logFile: - description: LogFile sets the log file of the SRIOV CNI plugin logs. - If unset (default), this will log to stderr and thus to multus and - container runtime logs. + description: |- + LogFile sets the log file of the SRIOV CNI plugin logs. If unset (default), this will log to stderr and thus + to multus and container runtime logs. type: string logLevel: default: info - description: LogLevel sets the log level of the SRIOV CNI plugin - - either of panic, error, warning, info, debug. Defaults to info if - left blank. + description: |- + LogLevel sets the log level of the SRIOV CNI plugin - either of panic, error, warning, info, debug. Defaults + to info if left blank. enum: - panic - error @@ -72,8 +78,9 @@ spec: minimum: 0 type: integer metaPlugins: - description: MetaPluginsConfig configuration to be used in order to - chain metaplugins to the sriov interface returned by the operator. + description: |- + MetaPluginsConfig configuration to be used in order to chain metaplugins to the sriov interface returned + by the operator. type: string minTxRate: description: Minimum tx rate, in Mbps, for the VF. Defaults to 0 (no diff --git a/bundle/manifests/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml b/bundle/manifests/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml index b4a4b5e1e..a09e2df5c 100644 --- a/bundle/manifests/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml +++ b/bundle/manifests/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.14.0 creationTimestamp: null name: sriovoperatorconfigs.sriovnetwork.openshift.io spec: @@ -21,14 +21,19 @@ spec: API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -41,9 +46,9 @@ spec: description: NodeSelector selects the nodes to be configured type: object configurationMode: - description: 'Flag to enable the sriov-network-config-daemon to use - a systemd service to configure SR-IOV devices on boot Default mode: - daemon' + description: |- + Flag to enable the sriov-network-config-daemon to use a systemd service to configure SR-IOV devices on boot + Default mode: daemon enum: - daemon - systemd diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovibnetworks.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovibnetworks.yaml index d619d5362..4b4b44d92 100644 --- a/config/crd/bases/sriovnetwork.openshift.io_sriovibnetworks.yaml +++ b/config/crd/bases/sriovnetwork.openshift.io_sriovibnetworks.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.14.0 name: sriovibnetworks.sriovnetwork.openshift.io spec: group: sriovnetwork.openshift.io @@ -21,14 +20,19 @@ spec: description: SriovIBNetwork is the Schema for the sriovibnetworks API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -36,8 +40,9 @@ spec: description: SriovIBNetworkSpec defines the desired state of SriovIBNetwork properties: capabilities: - description: 'Capabilities to be configured for this network. Capabilities - supported: (infinibandGUID), e.g. ''{"infinibandGUID": true}''' + description: |- + Capabilities to be configured for this network. + Capabilities supported: (infinibandGUID), e.g. '{"infinibandGUID": true}' type: string ipam: description: IPAM configuration to be used for this network. @@ -50,8 +55,9 @@ spec: - disable type: string metaPlugins: - description: MetaPluginsConfig configuration to be used in order to - chain metaplugins to the sriov interface returned by the operator. + description: |- + MetaPluginsConfig configuration to be used in order to chain metaplugins to the sriov interface returned + by the operator. type: string networkNamespace: description: Namespace of the NetworkAttachmentDefinition custom resource diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml index 84da2dee3..0508a326d 100644 --- a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml +++ b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.14.0 name: sriovnetworknodepolicies.sriovnetwork.openshift.io spec: group: sriovnetwork.openshift.io @@ -22,14 +21,19 @@ spec: API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml index a68c16f4e..0fb83dc24 100644 --- a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml +++ b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworknodestates.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.14.0 name: sriovnetworknodestates.sriovnetwork.openshift.io spec: group: sriovnetwork.openshift.io @@ -35,14 +34,19 @@ spec: API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml index b81999976..2cb2ece31 100644 --- a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml +++ b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.14.0 name: sriovnetworkpoolconfigs.sriovnetwork.openshift.io spec: group: sriovnetwork.openshift.io @@ -22,14 +21,19 @@ spec: API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -40,13 +44,15 @@ spec: anyOf: - type: integer - type: string - description: "maxUnavailable defines either an integer number or percentage - of nodes in the pool that can go Unavailable during an update. \n - A value larger than 1 will mean multiple nodes going unavailable - during the update, which may affect your workload stress on the - remaining nodes. Drain will respect Pod Disruption Budgets (PDBs) - such as etcd quorum guards, even if maxUnavailable is greater than - one." + description: |- + maxUnavailable defines either an integer number or percentage + of nodes in the pool that can go Unavailable during an update. + + + A value larger than 1 will mean multiple nodes going unavailable during + the update, which may affect your workload stress on the remaining nodes. + Drain will respect Pod Disruption Budgets (PDBs) such as etcd quorum guards, + even if maxUnavailable is greater than one. x-kubernetes-int-or-string: true nodeSelector: description: nodeSelector specifies a label selector for Nodes @@ -55,24 +61,24 @@ spec: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the key - and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to - a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a strategic + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic merge patch. items: type: string @@ -85,22 +91,24 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object + x-kubernetes-map-type: atomic ovsHardwareOffloadConfig: description: OvsHardwareOffloadConfig describes the OVS HWOL configuration for selected Nodes properties: name: - description: 'Name is mandatory and must be unique. On Kubernetes: - Name is the name of OvsHardwareOffloadConfig On OpenShift: Name - is the name of MachineConfigPool to be enabled with OVS hardware - offload' + description: |- + Name is mandatory and must be unique. + On Kubernetes: + Name is the name of OvsHardwareOffloadConfig + On OpenShift: + Name is the name of MachineConfigPool to be enabled with OVS hardware offload type: string type: object type: object diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworks.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworks.yaml index 15e1bfd3f..e33b9a3ed 100644 --- a/config/crd/bases/sriovnetwork.openshift.io_sriovnetworks.yaml +++ b/config/crd/bases/sriovnetwork.openshift.io_sriovnetworks.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.14.0 name: sriovnetworks.sriovnetwork.openshift.io spec: group: sriovnetwork.openshift.io @@ -21,14 +20,19 @@ spec: description: SriovNetwork is the Schema for the sriovnetworks API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -36,8 +40,9 @@ spec: description: SriovNetworkSpec defines the desired state of SriovNetwork properties: capabilities: - description: 'Capabilities to be configured for this network. Capabilities - supported: (mac|ips), e.g. ''{"mac": true}''' + description: |- + Capabilities to be configured for this network. + Capabilities supported: (mac|ips), e.g. '{"mac": true}' type: string ipam: description: IPAM configuration to be used for this network. @@ -50,15 +55,15 @@ spec: - disable type: string logFile: - description: LogFile sets the log file of the SRIOV CNI plugin logs. - If unset (default), this will log to stderr and thus to multus and - container runtime logs. + description: |- + LogFile sets the log file of the SRIOV CNI plugin logs. If unset (default), this will log to stderr and thus + to multus and container runtime logs. type: string logLevel: default: info - description: LogLevel sets the log level of the SRIOV CNI plugin - - either of panic, error, warning, info, debug. Defaults to info if - left blank. + description: |- + LogLevel sets the log level of the SRIOV CNI plugin - either of panic, error, warning, info, debug. Defaults + to info if left blank. enum: - panic - error @@ -73,8 +78,9 @@ spec: minimum: 0 type: integer metaPlugins: - description: MetaPluginsConfig configuration to be used in order to - chain metaplugins to the sriov interface returned by the operator. + description: |- + MetaPluginsConfig configuration to be used in order to chain metaplugins to the sriov interface returned + by the operator. type: string minTxRate: description: Minimum tx rate, in Mbps, for the VF. Defaults to 0 (no diff --git a/config/crd/bases/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml b/config/crd/bases/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml index 74b7752ab..5d944910d 100644 --- a/config/crd/bases/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml +++ b/config/crd/bases/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.14.0 name: sriovoperatorconfigs.sriovnetwork.openshift.io spec: group: sriovnetwork.openshift.io @@ -22,14 +21,19 @@ spec: API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -42,9 +46,9 @@ spec: description: NodeSelector selects the nodes to be configured type: object configurationMode: - description: 'Flag to enable the sriov-network-config-daemon to use - a systemd service to configure SR-IOV devices on boot Default mode: - daemon' + description: |- + Flag to enable the sriov-network-config-daemon to use a systemd service to configure SR-IOV devices on boot + Default mode: daemon enum: - daemon - systemd diff --git a/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovibnetworks.yaml b/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovibnetworks.yaml index d619d5362..4b4b44d92 100644 --- a/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovibnetworks.yaml +++ b/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovibnetworks.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.14.0 name: sriovibnetworks.sriovnetwork.openshift.io spec: group: sriovnetwork.openshift.io @@ -21,14 +20,19 @@ spec: description: SriovIBNetwork is the Schema for the sriovibnetworks API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -36,8 +40,9 @@ spec: description: SriovIBNetworkSpec defines the desired state of SriovIBNetwork properties: capabilities: - description: 'Capabilities to be configured for this network. Capabilities - supported: (infinibandGUID), e.g. ''{"infinibandGUID": true}''' + description: |- + Capabilities to be configured for this network. + Capabilities supported: (infinibandGUID), e.g. '{"infinibandGUID": true}' type: string ipam: description: IPAM configuration to be used for this network. @@ -50,8 +55,9 @@ spec: - disable type: string metaPlugins: - description: MetaPluginsConfig configuration to be used in order to - chain metaplugins to the sriov interface returned by the operator. + description: |- + MetaPluginsConfig configuration to be used in order to chain metaplugins to the sriov interface returned + by the operator. type: string networkNamespace: description: Namespace of the NetworkAttachmentDefinition custom resource diff --git a/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml b/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml index 84da2dee3..0508a326d 100644 --- a/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml +++ b/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.14.0 name: sriovnetworknodepolicies.sriovnetwork.openshift.io spec: group: sriovnetwork.openshift.io @@ -22,14 +21,19 @@ spec: API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object diff --git a/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodestates.yaml b/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodestates.yaml index a68c16f4e..0fb83dc24 100644 --- a/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodestates.yaml +++ b/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodestates.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.14.0 name: sriovnetworknodestates.sriovnetwork.openshift.io spec: group: sriovnetwork.openshift.io @@ -35,14 +34,19 @@ spec: API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object diff --git a/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml b/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml index b81999976..2cb2ece31 100644 --- a/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml +++ b/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.14.0 name: sriovnetworkpoolconfigs.sriovnetwork.openshift.io spec: group: sriovnetwork.openshift.io @@ -22,14 +21,19 @@ spec: API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -40,13 +44,15 @@ spec: anyOf: - type: integer - type: string - description: "maxUnavailable defines either an integer number or percentage - of nodes in the pool that can go Unavailable during an update. \n - A value larger than 1 will mean multiple nodes going unavailable - during the update, which may affect your workload stress on the - remaining nodes. Drain will respect Pod Disruption Budgets (PDBs) - such as etcd quorum guards, even if maxUnavailable is greater than - one." + description: |- + maxUnavailable defines either an integer number or percentage + of nodes in the pool that can go Unavailable during an update. + + + A value larger than 1 will mean multiple nodes going unavailable during + the update, which may affect your workload stress on the remaining nodes. + Drain will respect Pod Disruption Budgets (PDBs) such as etcd quorum guards, + even if maxUnavailable is greater than one. x-kubernetes-int-or-string: true nodeSelector: description: nodeSelector specifies a label selector for Nodes @@ -55,24 +61,24 @@ spec: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the key - and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to - a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a strategic + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic merge patch. items: type: string @@ -85,22 +91,24 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object + x-kubernetes-map-type: atomic ovsHardwareOffloadConfig: description: OvsHardwareOffloadConfig describes the OVS HWOL configuration for selected Nodes properties: name: - description: 'Name is mandatory and must be unique. On Kubernetes: - Name is the name of OvsHardwareOffloadConfig On OpenShift: Name - is the name of MachineConfigPool to be enabled with OVS hardware - offload' + description: |- + Name is mandatory and must be unique. + On Kubernetes: + Name is the name of OvsHardwareOffloadConfig + On OpenShift: + Name is the name of MachineConfigPool to be enabled with OVS hardware offload type: string type: object type: object diff --git a/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworks.yaml b/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworks.yaml index 15e1bfd3f..e33b9a3ed 100644 --- a/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworks.yaml +++ b/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworks.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.14.0 name: sriovnetworks.sriovnetwork.openshift.io spec: group: sriovnetwork.openshift.io @@ -21,14 +20,19 @@ spec: description: SriovNetwork is the Schema for the sriovnetworks API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -36,8 +40,9 @@ spec: description: SriovNetworkSpec defines the desired state of SriovNetwork properties: capabilities: - description: 'Capabilities to be configured for this network. Capabilities - supported: (mac|ips), e.g. ''{"mac": true}''' + description: |- + Capabilities to be configured for this network. + Capabilities supported: (mac|ips), e.g. '{"mac": true}' type: string ipam: description: IPAM configuration to be used for this network. @@ -50,15 +55,15 @@ spec: - disable type: string logFile: - description: LogFile sets the log file of the SRIOV CNI plugin logs. - If unset (default), this will log to stderr and thus to multus and - container runtime logs. + description: |- + LogFile sets the log file of the SRIOV CNI plugin logs. If unset (default), this will log to stderr and thus + to multus and container runtime logs. type: string logLevel: default: info - description: LogLevel sets the log level of the SRIOV CNI plugin - - either of panic, error, warning, info, debug. Defaults to info if - left blank. + description: |- + LogLevel sets the log level of the SRIOV CNI plugin - either of panic, error, warning, info, debug. Defaults + to info if left blank. enum: - panic - error @@ -73,8 +78,9 @@ spec: minimum: 0 type: integer metaPlugins: - description: MetaPluginsConfig configuration to be used in order to - chain metaplugins to the sriov interface returned by the operator. + description: |- + MetaPluginsConfig configuration to be used in order to chain metaplugins to the sriov interface returned + by the operator. type: string minTxRate: description: Minimum tx rate, in Mbps, for the VF. Defaults to 0 (no diff --git a/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml b/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml index 74b7752ab..5d944910d 100644 --- a/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml +++ b/deployment/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.14.0 name: sriovoperatorconfigs.sriovnetwork.openshift.io spec: group: sriovnetwork.openshift.io @@ -22,14 +21,19 @@ spec: API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -42,9 +46,9 @@ spec: description: NodeSelector selects the nodes to be configured type: object configurationMode: - description: 'Flag to enable the sriov-network-config-daemon to use - a systemd service to configure SR-IOV devices on boot Default mode: - daemon' + description: |- + Flag to enable the sriov-network-config-daemon to use a systemd service to configure SR-IOV devices on boot + Default mode: daemon enum: - daemon - systemd diff --git a/manifests/stable/sriov-network-operator.clusterserviceversion.yaml b/manifests/stable/sriov-network-operator.clusterserviceversion.yaml index bbdff4d5b..fa5880465 100644 --- a/manifests/stable/sriov-network-operator.clusterserviceversion.yaml +++ b/manifests/stable/sriov-network-operator.clusterserviceversion.yaml @@ -100,7 +100,7 @@ metadata: categories: Networking certified: "false" containerImage: quay.io/openshift/origin-sriov-network-operator:4.16 - createdAt: "2024-04-18T13:41:29Z" + createdAt: "2024-05-07T23:50:52Z" description: An operator for configuring SR-IOV components and initializing SRIOV network devices in Openshift cluster. features.operators.openshift.io/cnf: "false" diff --git a/manifests/stable/sriovnetwork.openshift.io_sriovibnetworks.yaml b/manifests/stable/sriovnetwork.openshift.io_sriovibnetworks.yaml index fa5ea4c75..c8eb80e6b 100644 --- a/manifests/stable/sriovnetwork.openshift.io_sriovibnetworks.yaml +++ b/manifests/stable/sriovnetwork.openshift.io_sriovibnetworks.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.14.0 creationTimestamp: null name: sriovibnetworks.sriovnetwork.openshift.io spec: @@ -20,14 +20,19 @@ spec: description: SriovIBNetwork is the Schema for the sriovibnetworks API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -35,8 +40,9 @@ spec: description: SriovIBNetworkSpec defines the desired state of SriovIBNetwork properties: capabilities: - description: 'Capabilities to be configured for this network. Capabilities - supported: (infinibandGUID), e.g. ''{"infinibandGUID": true}''' + description: |- + Capabilities to be configured for this network. + Capabilities supported: (infinibandGUID), e.g. '{"infinibandGUID": true}' type: string ipam: description: IPAM configuration to be used for this network. @@ -49,8 +55,9 @@ spec: - disable type: string metaPlugins: - description: MetaPluginsConfig configuration to be used in order to - chain metaplugins to the sriov interface returned by the operator. + description: |- + MetaPluginsConfig configuration to be used in order to chain metaplugins to the sriov interface returned + by the operator. type: string networkNamespace: description: Namespace of the NetworkAttachmentDefinition custom resource diff --git a/manifests/stable/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml b/manifests/stable/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml index 2de8005b8..e3af6cc47 100644 --- a/manifests/stable/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml +++ b/manifests/stable/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.14.0 creationTimestamp: null name: sriovnetworknodepolicies.sriovnetwork.openshift.io spec: @@ -21,14 +21,19 @@ spec: API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object diff --git a/manifests/stable/sriovnetwork.openshift.io_sriovnetworknodestates.yaml b/manifests/stable/sriovnetwork.openshift.io_sriovnetworknodestates.yaml index db2990a3c..55cdde2ed 100644 --- a/manifests/stable/sriovnetwork.openshift.io_sriovnetworknodestates.yaml +++ b/manifests/stable/sriovnetwork.openshift.io_sriovnetworknodestates.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.14.0 creationTimestamp: null name: sriovnetworknodestates.sriovnetwork.openshift.io spec: @@ -34,14 +34,19 @@ spec: API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object diff --git a/manifests/stable/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml b/manifests/stable/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml index a66384769..49f9cf60d 100644 --- a/manifests/stable/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml +++ b/manifests/stable/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.14.0 creationTimestamp: null name: sriovnetworkpoolconfigs.sriovnetwork.openshift.io spec: @@ -21,14 +21,19 @@ spec: API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -39,13 +44,15 @@ spec: anyOf: - type: integer - type: string - description: "maxUnavailable defines either an integer number or percentage - of nodes in the pool that can go Unavailable during an update. \n - A value larger than 1 will mean multiple nodes going unavailable - during the update, which may affect your workload stress on the - remaining nodes. Drain will respect Pod Disruption Budgets (PDBs) - such as etcd quorum guards, even if maxUnavailable is greater than - one." + description: |- + maxUnavailable defines either an integer number or percentage + of nodes in the pool that can go Unavailable during an update. + + + A value larger than 1 will mean multiple nodes going unavailable during + the update, which may affect your workload stress on the remaining nodes. + Drain will respect Pod Disruption Budgets (PDBs) such as etcd quorum guards, + even if maxUnavailable is greater than one. x-kubernetes-int-or-string: true nodeSelector: description: nodeSelector specifies a label selector for Nodes @@ -54,24 +61,24 @@ spec: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the key - and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship to - a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a strategic + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic merge patch. items: type: string @@ -84,22 +91,24 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object + x-kubernetes-map-type: atomic ovsHardwareOffloadConfig: description: OvsHardwareOffloadConfig describes the OVS HWOL configuration for selected Nodes properties: name: - description: 'Name is mandatory and must be unique. On Kubernetes: - Name is the name of OvsHardwareOffloadConfig On OpenShift: Name - is the name of MachineConfigPool to be enabled with OVS hardware - offload' + description: |- + Name is mandatory and must be unique. + On Kubernetes: + Name is the name of OvsHardwareOffloadConfig + On OpenShift: + Name is the name of MachineConfigPool to be enabled with OVS hardware offload type: string type: object type: object diff --git a/manifests/stable/sriovnetwork.openshift.io_sriovnetworks.yaml b/manifests/stable/sriovnetwork.openshift.io_sriovnetworks.yaml index 0c1034923..067e53a7f 100644 --- a/manifests/stable/sriovnetwork.openshift.io_sriovnetworks.yaml +++ b/manifests/stable/sriovnetwork.openshift.io_sriovnetworks.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.14.0 creationTimestamp: null name: sriovnetworks.sriovnetwork.openshift.io spec: @@ -20,14 +20,19 @@ spec: description: SriovNetwork is the Schema for the sriovnetworks API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -35,8 +40,9 @@ spec: description: SriovNetworkSpec defines the desired state of SriovNetwork properties: capabilities: - description: 'Capabilities to be configured for this network. Capabilities - supported: (mac|ips), e.g. ''{"mac": true}''' + description: |- + Capabilities to be configured for this network. + Capabilities supported: (mac|ips), e.g. '{"mac": true}' type: string ipam: description: IPAM configuration to be used for this network. @@ -49,15 +55,15 @@ spec: - disable type: string logFile: - description: LogFile sets the log file of the SRIOV CNI plugin logs. - If unset (default), this will log to stderr and thus to multus and - container runtime logs. + description: |- + LogFile sets the log file of the SRIOV CNI plugin logs. If unset (default), this will log to stderr and thus + to multus and container runtime logs. type: string logLevel: default: info - description: LogLevel sets the log level of the SRIOV CNI plugin - - either of panic, error, warning, info, debug. Defaults to info if - left blank. + description: |- + LogLevel sets the log level of the SRIOV CNI plugin - either of panic, error, warning, info, debug. Defaults + to info if left blank. enum: - panic - error @@ -72,8 +78,9 @@ spec: minimum: 0 type: integer metaPlugins: - description: MetaPluginsConfig configuration to be used in order to - chain metaplugins to the sriov interface returned by the operator. + description: |- + MetaPluginsConfig configuration to be used in order to chain metaplugins to the sriov interface returned + by the operator. type: string minTxRate: description: Minimum tx rate, in Mbps, for the VF. Defaults to 0 (no diff --git a/manifests/stable/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml b/manifests/stable/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml index b4a4b5e1e..a09e2df5c 100644 --- a/manifests/stable/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml +++ b/manifests/stable/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.14.0 creationTimestamp: null name: sriovoperatorconfigs.sriovnetwork.openshift.io spec: @@ -21,14 +21,19 @@ spec: API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -41,9 +46,9 @@ spec: description: NodeSelector selects the nodes to be configured type: object configurationMode: - description: 'Flag to enable the sriov-network-config-daemon to use - a systemd service to configure SR-IOV devices on boot Default mode: - daemon' + description: |- + Flag to enable the sriov-network-config-daemon to use a systemd service to configure SR-IOV devices on boot + Default mode: daemon enum: - daemon - systemd