From 0e9cb5df4f819c15fece6fae9d9f236c62d57ffb Mon Sep 17 00:00:00 2001 From: asifdxtreme Date: Tue, 11 Jun 2019 15:12:16 +0530 Subject: [PATCH 1/6] Bump volcano-sh/scheduler --- Gopkg.lock | 4 +- .../kube-batch/cmd/kube-batch/app/server.go | 2 +- .../kube-batch/pkg/scheduler/api/node_info.go | 88 +++++++++++++++---- .../kube-batch/pkg/scheduler/api/types.go | 21 +++++ .../kube-batch/pkg/scheduler/cache/cache.go | 11 ++- .../pkg/scheduler/cache/event_handlers.go | 12 +-- 6 files changed, 106 insertions(+), 32 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 8165d21a8f..9560e925f9 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -204,7 +204,7 @@ [[projects]] branch = "master" - digest = "1:4281fb8a5ed4671cd7e80c85599e00912f796819716abaefd697797ec9eb4491" + digest = "1:ea789d69109979142a6fbd569095f7678ea62b633b9b5a284f2fd1ff673da48f" name = "github.com/kubernetes-sigs/kube-batch" packages = [ "cmd/kube-batch/app", @@ -247,7 +247,7 @@ "pkg/version", ] pruneopts = "UT" - revision = "700381278a3e7da396507055d371dd93c1724322" + revision = "4b391ab34b53779e47243217006d8772cb86d8d8" source = "https://github.com/volcano-sh/scheduler" [[projects]] diff --git a/vendor/github.com/kubernetes-sigs/kube-batch/cmd/kube-batch/app/server.go b/vendor/github.com/kubernetes-sigs/kube-batch/cmd/kube-batch/app/server.go index f0e1491d72..1fbe6f415e 100644 --- a/vendor/github.com/kubernetes-sigs/kube-batch/cmd/kube-batch/app/server.go +++ b/vendor/github.com/kubernetes-sigs/kube-batch/cmd/kube-batch/app/server.go @@ -103,7 +103,7 @@ func Run(opt *options.ServerOption) error { // Prepare event clients. broadcaster := record.NewBroadcaster() broadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: leaderElectionClient.CoreV1().Events(opt.LockObjectNamespace)}) - eventRecorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "kube-batch"}) + eventRecorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: opt.SchedulerName}) hostname, err := os.Hostname() if err != nil { diff --git a/vendor/github.com/kubernetes-sigs/kube-batch/pkg/scheduler/api/node_info.go b/vendor/github.com/kubernetes-sigs/kube-batch/pkg/scheduler/api/node_info.go index 16257d4140..679818db51 100644 --- a/vendor/github.com/kubernetes-sigs/kube-batch/pkg/scheduler/api/node_info.go +++ b/vendor/github.com/kubernetes-sigs/kube-batch/pkg/scheduler/api/node_info.go @@ -19,6 +19,8 @@ package api import ( "fmt" + "github.com/golang/glog" + v1 "k8s.io/api/core/v1" ) @@ -27,6 +29,9 @@ type NodeInfo struct { Name string Node *v1.Node + // The state of node + State NodeState + // The releasing resource on that node Releasing *Resource // The idle resource on that node @@ -44,10 +49,18 @@ type NodeInfo struct { Other interface{} } +// NodeState defines the current state of node. +type NodeState struct { + Phase NodePhase + Reason string +} + // NewNodeInfo is used to create new nodeInfo object func NewNodeInfo(node *v1.Node) *NodeInfo { + var ni *NodeInfo + if node == nil { - return &NodeInfo{ + ni = &NodeInfo{ Releasing: EmptyResource(), Idle: EmptyResource(), Used: EmptyResource(), @@ -57,21 +70,25 @@ func NewNodeInfo(node *v1.Node) *NodeInfo { Tasks: make(map[TaskID]*TaskInfo), } - } - - return &NodeInfo{ - Name: node.Name, - Node: node, + } else { + ni = &NodeInfo{ + Name: node.Name, + Node: node, - Releasing: EmptyResource(), - Idle: NewResource(node.Status.Allocatable), - Used: EmptyResource(), + Releasing: EmptyResource(), + Idle: NewResource(node.Status.Allocatable), + Used: EmptyResource(), - Allocatable: NewResource(node.Status.Allocatable), - Capability: NewResource(node.Status.Capacity), + Allocatable: NewResource(node.Status.Allocatable), + Capability: NewResource(node.Status.Capacity), - Tasks: make(map[TaskID]*TaskInfo), + Tasks: make(map[TaskID]*TaskInfo), + } } + + ni.setNodeState(node) + + return ni } // Clone used to clone nodeInfo Object @@ -85,8 +102,47 @@ func (ni *NodeInfo) Clone() *NodeInfo { return res } +// Ready returns whether node is ready for scheduling +func (ni *NodeInfo) Ready() bool { + return ni.State.Phase == Ready +} + +func (ni *NodeInfo) setNodeState(node *v1.Node) { + // If node is nil, the node is un-initialized in cache + if node == nil { + ni.State = NodeState{ + Phase: NotReady, + Reason: "UnInitialized", + } + return + } + + // set NodeState according to resources + if !ni.Used.LessEqual(NewResource(node.Status.Allocatable)) { + ni.State = NodeState{ + Phase: NotReady, + Reason: "OutOfSync", + } + return + } + + // Node is ready (ignore node conditions because of taint/toleration) + ni.State = NodeState{ + Phase: Ready, + Reason: "", + } +} + // SetNode sets kubernetes node object to nodeInfo object func (ni *NodeInfo) SetNode(node *v1.Node) { + ni.setNodeState(node) + + if !ni.Ready() { + glog.Warningf("Failed to set node info, phase: %s, reason: %s", + ni.State.Phase, ni.State.Reason) + return + } + ni.Name = node.Name ni.Node = node @@ -176,16 +232,16 @@ func (ni *NodeInfo) UpdateTask(ti *TaskInfo) error { // String returns nodeInfo details in string format func (ni NodeInfo) String() string { - res := "" + tasks := "" i := 0 for _, task := range ni.Tasks { - res = res + fmt.Sprintf("\n\t %d: %v", i, task) + tasks = tasks + fmt.Sprintf("\n\t %d: %v", i, task) i++ } - return fmt.Sprintf("Node (%s): idle <%v>, used <%v>, releasing <%v>, taints <%v>%s", - ni.Name, ni.Idle, ni.Used, ni.Releasing, ni.Node.Spec.Taints, res) + return fmt.Sprintf("Node (%s): idle <%v>, used <%v>, releasing <%v>, state , taints <%v>%s", + ni.Name, ni.Idle, ni.Used, ni.Releasing, ni.State.Phase, ni.State.Reason, ni.Node.Spec.Taints, tasks) } diff --git a/vendor/github.com/kubernetes-sigs/kube-batch/pkg/scheduler/api/types.go b/vendor/github.com/kubernetes-sigs/kube-batch/pkg/scheduler/api/types.go index 97a583df6c..a88439e451 100644 --- a/vendor/github.com/kubernetes-sigs/kube-batch/pkg/scheduler/api/types.go +++ b/vendor/github.com/kubernetes-sigs/kube-batch/pkg/scheduler/api/types.go @@ -78,6 +78,27 @@ func (ts TaskStatus) String() string { } } +// NodePhase defines the phase of node +type NodePhase int + +const ( + // Ready means the node is ready for scheduling + Ready NodePhase = 1 << iota + // NotReady means the node is not ready for scheduling + NotReady +) + +func (np NodePhase) String() string { + switch np { + case Ready: + return "Ready" + case NotReady: + return "NotReady" + } + + return "Unknown" +} + // validateStatusUpdate validates whether the status transfer is valid. func validateStatusUpdate(oldStatus, newStatus TaskStatus) error { return nil diff --git a/vendor/github.com/kubernetes-sigs/kube-batch/pkg/scheduler/cache/cache.go b/vendor/github.com/kubernetes-sigs/kube-batch/pkg/scheduler/cache/cache.go index f6f1c04d2e..322d3b4d02 100644 --- a/vendor/github.com/kubernetes-sigs/kube-batch/pkg/scheduler/cache/cache.go +++ b/vendor/github.com/kubernetes-sigs/kube-batch/pkg/scheduler/cache/cache.go @@ -200,7 +200,7 @@ func newSchedulerCache(config *rest.Config, schedulerName string, defaultQueue s // Prepare event clients. broadcaster := record.NewBroadcaster() broadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: sc.kubeclient.CoreV1().Events("")}) - sc.Recorder = broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "kube-batch"}) + sc.Recorder = broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: schedulerName}) sc.Binder = &defaultBinder{ kubeclient: sc.kubeclient, @@ -465,7 +465,10 @@ func (sc *SchedulerCache) taskUnschedulable(task *api.TaskInfo, message string) pod := task.Pod.DeepCopy() - sc.Recorder.Eventf(pod, v1.EventTypeWarning, string(v1.PodReasonUnschedulable), message) + // The reason field in 'Events' should be "FailedScheduling", there is not constants defined for this in + // k8s core, so using the same string here. + // The reason field in PodCondition should be "Unschedulable" + sc.Recorder.Eventf(pod, v1.EventTypeWarning, "FailedScheduling", message) if _, err := sc.StatusUpdater.UpdatePodCondition(pod, &v1.PodCondition{ Type: v1.PodScheduled, Status: v1.ConditionFalse, @@ -546,6 +549,10 @@ func (sc *SchedulerCache) Snapshot() *kbapi.ClusterInfo { } for _, value := range sc.Nodes { + if !value.Ready() { + continue + } + snapshot.Nodes[value.Name] = value.Clone() } diff --git a/vendor/github.com/kubernetes-sigs/kube-batch/pkg/scheduler/cache/event_handlers.go b/vendor/github.com/kubernetes-sigs/kube-batch/pkg/scheduler/cache/event_handlers.go index 4260521fdc..3b005a2fee 100644 --- a/vendor/github.com/kubernetes-sigs/kube-batch/pkg/scheduler/cache/event_handlers.go +++ b/vendor/github.com/kubernetes-sigs/kube-batch/pkg/scheduler/cache/event_handlers.go @@ -18,7 +18,6 @@ package cache import ( "fmt" - "reflect" "github.com/golang/glog" @@ -269,19 +268,10 @@ func (sc *SchedulerCache) addNode(node *v1.Node) error { return nil } -func isNodeInfoUpdated(oldNode, newNode *v1.Node) bool { - return !reflect.DeepEqual(oldNode.Status.Allocatable, newNode.Status.Allocatable) || - !reflect.DeepEqual(oldNode.Spec.Taints, newNode.Spec.Taints) || - !reflect.DeepEqual(oldNode.Labels, newNode.Labels) || - !reflect.DeepEqual(oldNode.Spec.Unschedulable, newNode.Spec.Unschedulable) -} - // Assumes that lock is already acquired. func (sc *SchedulerCache) updateNode(oldNode, newNode *v1.Node) error { if sc.Nodes[newNode.Name] != nil { - if isNodeInfoUpdated(oldNode, newNode) { - sc.Nodes[newNode.Name].SetNode(newNode) - } + sc.Nodes[newNode.Name].SetNode(newNode) return nil } From ce5600f320d84c13a0dd79a3e544b04773516770 Mon Sep 17 00:00:00 2001 From: Thandayuthapani Date: Wed, 12 Jun 2019 16:11:54 +0530 Subject: [PATCH 2/6] Fix E2E because of Event Change --- test/e2e/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/util.go b/test/e2e/util.go index 2680568b7e..03af6c8beb 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -423,7 +423,7 @@ func jobUnschedulable(ctx *context, job *vkv1.Job, now time.Time) error { for _, event := range events.Items { target := event.InvolvedObject if strings.HasPrefix(target.Name, pg.Name) && target.Namespace == pg.Namespace { - if event.Reason == string("Unschedulable") && event.LastTimestamp.After(now) { + if event.Reason == string("Unschedulable") || event.Reason == string("FailedScheduling") && event.LastTimestamp.After(now) { return true, nil } } From 2155b6042af3030969c2869bb8f31d50098b6f08 Mon Sep 17 00:00:00 2001 From: fangyuan Date: Wed, 12 Jun 2019 22:24:51 +0800 Subject: [PATCH 3/6] Fix some typos in documents --- README.md | 2 +- docs/design/drf - fairshare.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 677d5c2563..6417c26465 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Some examples of the mechanisms and features that Volcano adds to Kubernetes are 2. Fair-share scheduling 3. Queue scheduling 4. Preemption and reclaims - 5. Reservartions and backfills + 5. Reservations and backfills 6. Topology-based scheduling 3. Runtime extensions, e.g: 1. Support for specialized continer runtimes like Singularity, diff --git a/docs/design/drf - fairshare.md b/docs/design/drf - fairshare.md index 4b69448f97..038b60b5b4 100644 --- a/docs/design/drf - fairshare.md +++ b/docs/design/drf - fairshare.md @@ -175,5 +175,5 @@ All these plugin would choose some victims respective, and the intersection of t | queue | namespace | requested | queue assigned | namespace assigned | | ----- | --------- | --------- | -------------- | ------------------ | | q1 w1 | ns1 w2 | | 4 cpu | | - | q2 w3 | na1 w2 | 5 cpu | 12 cpu | 3 cpu | + | q2 w3 | ns1 w2 | 5 cpu | 12 cpu | 3 cpu | | | ns2 w6 | 20 cpu | | 9 cpu | \ No newline at end of file From c644aa5d5f19dfe187b185898c03713f5782a3bb Mon Sep 17 00:00:00 2001 From: fangyuan Date: Wed, 12 Jun 2019 23:31:34 +0800 Subject: [PATCH 4/6] fix typos in codes --- pkg/cli/job/delete.go | 3 ++- pkg/cli/job/resume.go | 3 ++- pkg/cli/job/suspend.go | 2 +- pkg/cli/job/view.go | 2 +- pkg/cli/queue/get.go | 2 +- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pkg/cli/job/delete.go b/pkg/cli/job/delete.go index e6f697b500..05789818ec 100644 --- a/pkg/cli/job/delete.go +++ b/pkg/cli/job/delete.go @@ -18,6 +18,7 @@ package job import ( "fmt" + "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -48,7 +49,7 @@ func DeleteJob() error { } if deleteJobFlags.JobName == "" { - err := fmt.Errorf("job name is mandaorty to delete a particular job") + err := fmt.Errorf("job name is mandatory to delete a particular job") return err } diff --git a/pkg/cli/job/resume.go b/pkg/cli/job/resume.go index d73c1d6dcd..8779bd70e1 100644 --- a/pkg/cli/job/resume.go +++ b/pkg/cli/job/resume.go @@ -17,6 +17,7 @@ package job import ( "fmt" + "github.com/spf13/cobra" "volcano.sh/volcano/pkg/apis/batch/v1alpha1" @@ -44,7 +45,7 @@ func ResumeJob() error { return err } if resumeJobFlags.JobName == "" { - err := fmt.Errorf("job name is mandaorty to resume a particular job") + err := fmt.Errorf("job name is mandatory to resume a particular job") return err } diff --git a/pkg/cli/job/suspend.go b/pkg/cli/job/suspend.go index 504cdd2381..fb0ebf7fd6 100644 --- a/pkg/cli/job/suspend.go +++ b/pkg/cli/job/suspend.go @@ -45,7 +45,7 @@ func SuspendJob() error { } if suspendJobFlags.JobName == "" { - err := fmt.Errorf("job name is mandaorty to suspend a particular job") + err := fmt.Errorf("job name is mandatory to suspend a particular job") return err } diff --git a/pkg/cli/job/view.go b/pkg/cli/job/view.go index 65c11a37d5..a4d9209fc4 100644 --- a/pkg/cli/job/view.go +++ b/pkg/cli/job/view.go @@ -36,7 +36,7 @@ func ViewJob() error { return err } if viewJobFlags.JobName == "" { - err := fmt.Errorf("job name (specified by --name or -n) is mandaorty to view a particular job") + err := fmt.Errorf("job name (specified by --name or -n) is mandatory to view a particular job") return err } diff --git a/pkg/cli/queue/get.go b/pkg/cli/queue/get.go index bc5a256bcf..4b3c946e42 100644 --- a/pkg/cli/queue/get.go +++ b/pkg/cli/queue/get.go @@ -52,7 +52,7 @@ func GetQueue() error { } if getQueueFlags.Name == "" { - err := fmt.Errorf("name is mandaorty to get the particular queue details") + err := fmt.Errorf("name is mandatory to get the particular queue details") return err } From d4d4a36b899b30e41a7e55a7677f9425a077189c Mon Sep 17 00:00:00 2001 From: Rajadeepan D Ramesh Date: Mon, 17 Jun 2019 07:49:12 +0530 Subject: [PATCH 5/6] Added/Updated license for copied scripts --- hack/update-gencode.sh | 2 +- hack/verify-gofmt.sh | 14 ++++++++++++++ hack/verify-golint.sh | 14 ++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/hack/update-gencode.sh b/hack/update-gencode.sh index 0aaad3a1b3..bc727d1afe 100755 --- a/hack/update-gencode.sh +++ b/hack/update-gencode.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Volcano Authors. +# Copyright 2014 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/hack/verify-gofmt.sh b/hack/verify-gofmt.sh index 1c83594c81..460f0ca93f 100755 --- a/hack/verify-gofmt.sh +++ b/hack/verify-gofmt.sh @@ -1,5 +1,19 @@ #!/bin/bash +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + set -o errexit set -o nounset set -o pipefail diff --git a/hack/verify-golint.sh b/hack/verify-golint.sh index 188bf8cf3d..b6fc290d66 100755 --- a/hack/verify-golint.sh +++ b/hack/verify-golint.sh @@ -1,5 +1,19 @@ #!/bin/bash +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + set -o errexit set -o nounset set -o pipefail From 06e89cee55f845556a59d627c28722e3e3e1ea65 Mon Sep 17 00:00:00 2001 From: TommyLike Date: Mon, 17 Jun 2019 10:43:36 +0800 Subject: [PATCH 6/6] Fix gcp auth issue --- pkg/cli/queue/util.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/cli/queue/util.go b/pkg/cli/queue/util.go index 5081c3fb95..f6617f287d 100644 --- a/pkg/cli/queue/util.go +++ b/pkg/cli/queue/util.go @@ -21,6 +21,8 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + // Initialize client auth plugin. + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" ) func homeDir() string {