-
Notifications
You must be signed in to change notification settings - Fork 5.6k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
add TaskFail interface #2719
add TaskFail interface #2719
Changes from all commits
e25c155
52cc601
108b0fa
7663a40
a94d217
8f70885
578dd09
a40a7a5
b64c7a6
d05d19b
dd8685f
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -31,30 +31,36 @@ type Chunk struct { | |
Index recordio.Index // chunk index | ||
} | ||
|
||
// TaskMeta is a struct which stores task's meta info. | ||
type TaskMeta struct { | ||
ID int | ||
Epoch int | ||
} | ||
|
||
// Task is the basic unit of data instances assigned to trainers. | ||
type Task struct { | ||
ID int | ||
Meta TaskMeta | ||
Chunks []Chunk | ||
} | ||
|
||
type taskEntry struct { | ||
Epoch int | ||
NumTimeout int | ||
Task Task | ||
Task Task | ||
// A task fails if it's timeout or trainer reports it exits unnormally. | ||
NumFailure int | ||
} | ||
|
||
type taskQueues struct { | ||
Todo []taskEntry | ||
Pending map[int]taskEntry // map from task ID to task entry | ||
Done []taskEntry | ||
Failed []Task | ||
Failed []taskEntry | ||
} | ||
|
||
// Service is the master server service. | ||
type Service struct { | ||
chunksPerTask int | ||
timeoutDur time.Duration | ||
timeoutMax int | ||
failureMax int | ||
ready chan struct{} | ||
store Store | ||
|
||
|
@@ -73,7 +79,7 @@ func partition(chunks []Chunk, chunksPerTask int) []taskEntry { | |
var cur taskEntry | ||
for i, c := range chunks { | ||
if i%chunksPerTask == 0 && len(cur.Task.Chunks) > 0 { | ||
cur.Task.ID = id | ||
cur.Task.Meta.ID = id | ||
id++ | ||
result = append(result, cur) | ||
cur.Task.Chunks = nil | ||
|
@@ -83,19 +89,19 @@ func partition(chunks []Chunk, chunksPerTask int) []taskEntry { | |
} | ||
|
||
if len(cur.Task.Chunks) > 0 { | ||
cur.Task.ID = id | ||
cur.Task.Meta.ID = id | ||
result = append(result, cur) | ||
} | ||
|
||
return result | ||
} | ||
|
||
// NewService creates a new service. | ||
func NewService(store Store, chunksPerTask int, timeoutDur time.Duration, timeoutMax int) (*Service, error) { | ||
func NewService(store Store, chunksPerTask int, timeoutDur time.Duration, failureMax int) (*Service, error) { | ||
s := &Service{} | ||
s.chunksPerTask = chunksPerTask | ||
s.timeoutDur = timeoutDur | ||
s.timeoutMax = timeoutMax | ||
s.failureMax = failureMax | ||
s.taskQueues = taskQueues{} | ||
s.taskQueues.Pending = make(map[int]taskEntry) | ||
s.ready = make(chan struct{}) | ||
|
@@ -257,6 +263,34 @@ func (s *Service) SetDataset(globPaths []string, dummy *int) error { | |
return nil | ||
} | ||
|
||
func (s *Service) processFailedTask(t taskEntry, epoch int) { | ||
if t.Task.Meta.Epoch != epoch { | ||
// new epoch, task launched after the | ||
// schedule of this timeout check or failed status report. | ||
return | ||
} | ||
|
||
defer func() { | ||
err := s.snapshot() | ||
if err != nil { | ||
log.Errorln(err) | ||
} | ||
}() | ||
|
||
delete(s.taskQueues.Pending, t.Task.Meta.ID) | ||
|
||
t.NumFailure++ | ||
if t.NumFailure > s.failureMax { | ||
log.Warningf("Task %v failed %d times, discard.", t.Task, t.NumFailure) | ||
s.taskQueues.Failed = append(s.taskQueues.Failed, t) | ||
return | ||
} | ||
|
||
log.Warningf("Task %v failed %d times, discard.", t.Task, t.NumFailure) | ||
s.taskQueues.Todo = append(s.taskQueues.Todo, t) | ||
return | ||
} | ||
|
||
func (s *Service) checkTimeoutFunc(taskID int, epoch int) func() { | ||
return func() { | ||
s.mu.Lock() | ||
|
@@ -267,30 +301,7 @@ func (s *Service) checkTimeoutFunc(taskID int, epoch int) func() { | |
return | ||
} | ||
|
||
if t.Epoch != epoch { | ||
// new epoch, task launched after the | ||
// schedule of this timeout check. | ||
return | ||
} | ||
|
||
defer func() { | ||
err := s.snapshot() | ||
if err != nil { | ||
log.Errorln(err) | ||
} | ||
}() | ||
|
||
delete(s.taskQueues.Pending, t.Task.ID) | ||
|
||
t.NumTimeout++ | ||
if t.NumTimeout > s.timeoutMax { | ||
log.Warningf("Task %v timed out %d times, discard.", t.Task, t.NumTimeout) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 这里可能也会被failed调用,所以不一定都是time out,可以用泛化点的描述。 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done. |
||
s.taskQueues.Failed = append(s.taskQueues.Failed, t.Task) | ||
return | ||
} | ||
|
||
log.Warningf("Task %v timed out %d times, retry.", t.Task, t.NumTimeout) | ||
s.taskQueues.Todo = append(s.taskQueues.Todo, t) | ||
s.processFailedTask(t, epoch) | ||
} | ||
} | ||
|
||
|
@@ -339,18 +350,18 @@ func (s *Service) GetTask(dummy int, task *Task) error { | |
} | ||
|
||
t := s.taskQueues.Todo[0] | ||
t.Epoch++ | ||
t.Task.Meta.Epoch++ | ||
s.taskQueues.Todo = s.taskQueues.Todo[1:] | ||
s.taskQueues.Pending[t.Task.ID] = t | ||
s.taskQueues.Pending[t.Task.Meta.ID] = t | ||
err := s.snapshot() | ||
if err != nil { | ||
return err | ||
} | ||
|
||
*task = t.Task | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Delete unused There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 这行是返回值,是有用的。:) |
||
log.WithFields(s.logFields()).Infof("Task #%d dispatched.", task.ID) | ||
log.WithFields(s.logFields()).Infof("Task #%v dispatched.", t.Task.Meta) | ||
|
||
time.AfterFunc(s.timeoutDur, s.checkTimeoutFunc(t.Task.ID, t.Epoch)) | ||
time.AfterFunc(s.timeoutDur, s.checkTimeoutFunc(t.Task.Meta.ID, t.Task.Meta.Epoch)) | ||
return nil | ||
} | ||
|
||
|
@@ -365,13 +376,12 @@ func (s *Service) TaskFinished(taskID int, dummy *int) error { | |
|
||
t, ok := s.taskQueues.Pending[taskID] | ||
if !ok { | ||
err := errors.New("pending task not found") | ||
log.WithFields(s.logFields()).Warningln("Pending task #%d not found.", taskID) | ||
return err | ||
return nil | ||
} | ||
|
||
// task finished, reset timeout | ||
t.NumTimeout = 0 | ||
t.NumFailure = 0 | ||
s.taskQueues.Done = append(s.taskQueues.Done, t) | ||
delete(s.taskQueues.Pending, taskID) | ||
|
||
|
@@ -389,3 +399,22 @@ func (s *Service) TaskFinished(taskID int, dummy *int) error { | |
} | ||
return err | ||
} | ||
|
||
// TaskFailed tells the service that a task is failed. | ||
func (s *Service) TaskFailed(meta TaskMeta, dummy *int) error { | ||
select { | ||
case <-s.ready: | ||
} | ||
|
||
s.mu.Lock() | ||
defer s.mu.Unlock() | ||
|
||
t, ok := s.taskQueues.Pending[meta.ID] | ||
if !ok { | ||
log.WithFields(s.logFields()).Warningln("TaskFailed:Pending task #%v not found.", t.Task.Meta) | ||
return nil | ||
} | ||
|
||
s.processFailedTask(t, meta.Epoch) | ||
return nil | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
把
Task
改成了taskEntry
的原因是,我觉得Failed task
应该保留进入错误队列时候的上下文状态。