diff --git a/cluster/calcium/node.go b/cluster/calcium/node.go index 1f706930c..0c52d0d71 100644 --- a/cluster/calcium/node.go +++ b/cluster/calcium/node.go @@ -57,6 +57,40 @@ func (c *Calcium) GetNode(ctx context.Context, nodename string) (*types.Node, er return node, logger.Err(ctx, errors.WithStack(err)) } +func (c *Calcium) setAllWorkloadsOnNodeDown(ctx context.Context, nodename string) { + workloads, err := c.store.ListNodeWorkloads(ctx, nodename, nil) + if err != nil { + log.Errorf(ctx, "[setAllWorkloadsOnNodeDown] failed to list node workloads, node %v, err: %v", nodename, errors.WithStack(err)) + return + } + + for _, workload := range workloads { + appname, entrypoint, _, err := utils.ParseWorkloadName(workload.Name) + if err != nil { + log.Errorf(ctx, "[setAllWorkloadsOnNodeDown] Set workload %s on node %s as inactive failed %v", workload.ID, nodename, err) + continue + } + + if workload.StatusMeta == nil { + workload.StatusMeta = &types.StatusMeta{ID: workload.ID} + } + workload.StatusMeta.Running = false + workload.StatusMeta.Healthy = false + + // Set these attributes to set workload status + workload.StatusMeta.Appname = appname + workload.StatusMeta.Nodename = workload.Nodename + workload.StatusMeta.Entrypoint = entrypoint + + // mark workload which belongs to this node as unhealthy + if err = c.store.SetWorkloadStatus(ctx, workload.StatusMeta, 0); err != nil { + log.Errorf(ctx, "[SetNodeAvailable] Set workload %s on node %s as inactive failed %v", workload.ID, nodename, errors.WithStack(err)) + } else { + log.Infof(ctx, "[SetNodeAvailable] Set workload %s on node %s as inactive", workload.ID, nodename) + } + } +} + // SetNode set node available or not func (c *Calcium) SetNode(ctx context.Context, opts *types.SetNodeOptions) (*types.Node, error) { // nolint logger := log.WithField("Calcium", "SetNode").WithField("opts", opts) @@ -82,35 +116,7 @@ func (c *Calcium) SetNode(ctx context.Context, opts *types.SetNodeOptions) (*typ } } if opts.WorkloadsDown { - workloads, err := c.store.ListNodeWorkloads(ctx, opts.Nodename, nil) - if err != nil { - return logger.Err(ctx, errors.WithStack(err)) - } - for _, workload := range workloads { - appname, entrypoint, _, err := utils.ParseWorkloadName(workload.Name) - if err != nil { - log.Errorf(ctx, "[SetNodeAvailable] Set workload %s on node %s inactive failed %v", workload.ID, opts.Nodename, err) - continue - } - - if workload.StatusMeta == nil { - workload.StatusMeta = &types.StatusMeta{ID: workload.ID} - } - workload.StatusMeta.Running = false - workload.StatusMeta.Healthy = false - - // Set these attributes to set workload status - workload.StatusMeta.Appname = appname - workload.StatusMeta.Nodename = workload.Nodename - workload.StatusMeta.Entrypoint = entrypoint - - // mark workload which belongs to this node as unhealthy - if err = c.store.SetWorkloadStatus(ctx, workload.StatusMeta, 0); err != nil { - log.Errorf(ctx, "[SetNodeAvailable] Set workload %s on node %s as inactive failed %v", workload.ID, opts.Nodename, errors.WithStack(err)) - } else { - log.Infof(ctx, "[SetNodeAvailable] Set workload %s on node %s as inactive", workload.ID, opts.Nodename) - } - } + c.setAllWorkloadsOnNodeDown(ctx, opts.Nodename) } // update node endpoint if opts.Endpoint != "" { diff --git a/cluster/calcium/node_test.go b/cluster/calcium/node_test.go index c568d80db..73e1780eb 100644 --- a/cluster/calcium/node_test.go +++ b/cluster/calcium/node_test.go @@ -165,10 +165,10 @@ func TestSetNode(t *testing.T) { assert.Equal(t, n.Name, name) assert.Equal(t, n.Endpoint, "tcp://127.0.0.1:2379") // not available - // failed by list node workloads + // can still set node even if ListNodeWorkloads fails store.On("ListNodeWorkloads", mock.Anything, mock.Anything, mock.Anything).Return(nil, types.ErrNoETCD).Once() _, err = c.SetNode(ctx, &types.SetNodeOptions{Nodename: "test", StatusOpt: 0, WorkloadsDown: true}) - assert.Error(t, err) + assert.NoError(t, err) workloads := []*types.Workload{{Name: "wrong_name"}, {Name: "a_b_c"}} store.On("ListNodeWorkloads", mock.Anything, mock.Anything, mock.Anything).Return(workloads, nil) store.On("SetWorkloadStatus",