From 02624ff585ab09efe08e6b914b9cca2dd65e25e8 Mon Sep 17 00:00:00 2001 From: xcaspar Date: Tue, 12 Nov 2019 17:36:13 +0800 Subject: [PATCH] bugfix: fix the pod network experiment --- .../templates/deployment.yaml | 4 +- exec/model/context.go | 15 ++++- exec/model/executor.go | 61 ++++++++++-------- exec/pod/controller.go | 4 +- exec/pod/delete.go | 12 ++-- exec/pod/osexp.go | 62 ++++++++++++++----- 6 files changed, 102 insertions(+), 56 deletions(-) diff --git a/deploy/helm/chaosblade-operator/templates/deployment.yaml b/deploy/helm/chaosblade-operator/templates/deployment.yaml index dbcda6d..115d517 100644 --- a/deploy/helm/chaosblade-operator/templates/deployment.yaml +++ b/deploy/helm/chaosblade-operator/templates/deployment.yaml @@ -1,8 +1,6 @@ apiVersion: extensions/v1beta1 kind: DaemonSet metadata: - labels: - app: chaosblade-tool name: chaosblade-tool namespace: kube-system spec: @@ -14,6 +12,7 @@ spec: metadata: labels: app: chaosblade-tool + part-of: chaosblade name: chaosblade-tool spec: affinity: @@ -66,6 +65,7 @@ spec: metadata: labels: name: chaosblade-operator + part-of: chaosblade spec: serviceAccountName: chaosblade containers: diff --git a/exec/model/context.go b/exec/model/context.go index 07cd636..f0fe128 100644 --- a/exec/model/context.go +++ b/exec/model/context.go @@ -32,8 +32,8 @@ const ( NodeNameUidMapKey = "NodeNameUidMap" // nodeName: []{} NodeNameContainerObjectMetasMapKey = "NodeNameContainerObjectMetasMapKey" - //{Name: xx, Namespace: xx, Uid: xxx, NodeName: xxx} - PodObjectMetaKey = "PodObjectMeta" + //[{Name: xx, Namespace: xx, Uid: xxx, NodeName: xxx}] + PodObjectMetaListKey = "PodObjectMetaListKey" // For destroy operation // nodeName:[{uid:expId}, {uid:expId}] @@ -72,6 +72,8 @@ type ContainerObjectMeta struct { type NodeNameContainerObjectMetasMap map[string][]ContainerObjectMeta +type PodObjectMetaList []PodObjectMeta + func ExtractNodeNameUidMapFromContext(ctx context.Context) (NodeNameUidMap, error) { nodeNameUidMapValue := ctx.Value(NodeNameUidMapKey) if nodeNameUidMapValue == nil { @@ -99,6 +101,15 @@ func ExtractNodeNameContainerMetasMapFromContext(ctx context.Context) (NodeNameC return containerObjectMetas, nil } +func ExtractPodObjectMetasFromContext(ctx context.Context) (PodObjectMetaList, error) { + podObjectMetaValues := ctx.Value(PodObjectMetaListKey) + if podObjectMetaValues == nil { + return nil, fmt.Errorf("less pod object meta parameter") + } + podObjectMetas := podObjectMetaValues.(PodObjectMetaList) + return podObjectMetas, nil +} + func GetChaosBladePodListOptions() *client.ListOptions { return &client.ListOptions{ Namespace: meta.Constant.Namespace, diff --git a/exec/model/executor.go b/exec/model/executor.go index 78e0277..2154c15 100644 --- a/exec/model/executor.go +++ b/exec/model/executor.go @@ -39,6 +39,7 @@ type ExperimentIdentifier struct { Uid string Name string Command string + Error string } func NewExperimentIdentifier(id, uid, name, command string) ExperimentIdentifier { @@ -50,6 +51,15 @@ func NewExperimentIdentifier(id, uid, name, command string) ExperimentIdentifier } } +func NewExperimentIdentifierWithError(id, uid, name, error string) ExperimentIdentifier { + return ExperimentIdentifier{ + Id: id, + Uid: uid, + Name: name, + Error: error, + } +} + type ExecCommandInPodExecutor struct { Client *channel.Client CommandFunc func(ctx context.Context, model *spec.ExpModel, @@ -79,20 +89,14 @@ func (e *ExecCommandInPodExecutor) Exec(uid string, ctx context.Context, expMode // ResourceIdentifier is used to pass the necessary values in context type ResourceIdentifier struct { - NodeName string - NodeUid string - PodName string - PodUid string - ContainerId string + NodeName string + NodeUid string } -func NewResourceIdentifier(nodeName, nodeUid, podName, podUid, containerId string) *ResourceIdentifier { +func NewResourceIdentifier(nodeName, nodeUid string) *ResourceIdentifier { return &ResourceIdentifier{ - NodeName: nodeName, - NodeUid: nodeUid, - PodName: podName, - PodUid: podUid, - ContainerId: containerId, + NodeName: nodeName, + NodeUid: nodeUid, } } @@ -137,16 +141,14 @@ func (e *ExecCommandInPodExecutor) execInMatchedPod(ctx context.Context, nodeNam ctx = spec.SetDestroyFlag(ctx, expIds) } - targetPod := e.getExecPod(nodeName, podList) - if targetPod == nil { - rsStatus.CreateFailResourceStatus(fmt.Sprintf("can not find the target pod on %s node", nodeName)) + targetExcCommandPod := e.getExecPod(nodeName, podList) + if targetExcCommandPod == nil { + rsStatus.CreateFailResourceStatus(fmt.Sprintf("can not find the target exec pod on %s node", nodeName)) statuses = append(statuses, rsStatus) continue } - // get the first container from pod - containerId, _ := GetOneAvailableContainerIdFromPod(*targetPod) - resourceIdentifier := NewResourceIdentifier(nodeName, nodeUid, targetPod.Name, string(targetPod.UID), containerId) - success, statuses = e.execCommands(ctx, expModel, rsStatus, targetPod, statuses, resourceIdentifier) + resourceIdentifier := NewResourceIdentifier(nodeName, nodeUid) + success, statuses = e.execCommands(ctx, expModel, rsStatus, targetExcCommandPod, statuses, resourceIdentifier) } logrus.Infof("success: %t, statuses: %+v", success, statuses) if success { @@ -165,7 +167,7 @@ func (e *ExecCommandInPodExecutor) execInMatchedPod(ctx context.Context, nodeNam } func (e *ExecCommandInPodExecutor) execCommands(ctx context.Context, expModel *spec.ExpModel, - rsStatus v1alpha1.ResourceStatus, targetPod *v1.Pod, statuses []v1alpha1.ResourceStatus, + rsStatus v1alpha1.ResourceStatus, targetExecPod *v1.Pod, statuses []v1alpha1.ResourceStatus, resourceIdentifier *ResourceIdentifier) (bool, []v1alpha1.ResourceStatus) { success := false @@ -181,15 +183,20 @@ func (e *ExecCommandInPodExecutor) execCommands(ctx context.Context, expModel *s newStatus.Id = identifier.Id newStatus.Uid = identifier.Uid newStatus.Name = identifier.Name - response := e.Client.Exec(targetPod, meta.Constant.PodName, identifier.Command, time.Second*30) - if response.Success { - if _, ok := spec.IsDestroy(ctx); !ok { - newStatus.Id = response.Result.(string) - } - newStatus = newStatus.CreateSuccessResourceStatus() - success = true + + if identifier.Error != "" { + newStatus = newStatus.CreateFailResourceStatus(identifier.Error) } else { - newStatus = newStatus.CreateFailResourceStatus(response.Err) + response := e.Client.Exec(targetExecPod, meta.Constant.PodName, identifier.Command, time.Second*30) + if response.Success { + if _, ok := spec.IsDestroy(ctx); !ok { + newStatus.Id = response.Result.(string) + } + newStatus = newStatus.CreateSuccessResourceStatus() + success = true + } else { + newStatus = newStatus.CreateFailResourceStatus(response.Err) + } } statuses = append(statuses, newStatus) } diff --git a/exec/pod/controller.go b/exec/pod/controller.go index aa32a80..e575fe8 100644 --- a/exec/pod/controller.go +++ b/exec/pod/controller.go @@ -61,7 +61,7 @@ func (e *ExpController) Create(ctx context.Context, expSpec v1alpha1.ExperimentS } func setNecessaryObjectsToContext(ctx context.Context, pods []v1.Pod) context.Context { - podObjectMetas := make([]model.PodObjectMeta, 0) + podObjectMetas := model.PodObjectMetaList{} nodeNameUidMap := model.NodeNameUidMap{} for _, pod := range pods { podObjectMeta := model.PodObjectMeta{ @@ -71,7 +71,7 @@ func setNecessaryObjectsToContext(ctx context.Context, pods []v1.Pod) context.Co // node uid is unuseful for pod experiments nodeNameUidMap[pod.Spec.NodeName] = "" } - ctx = context.WithValue(ctx, model.PodObjectMetaKey, podObjectMetas) + ctx = context.WithValue(ctx, model.PodObjectMetaListKey, podObjectMetas) ctx = context.WithValue(ctx, model.NodeNameUidMapKey, nodeNameUidMap) return ctx } diff --git a/exec/pod/delete.go b/exec/pod/delete.go index d1cb92c..abf1f09 100644 --- a/exec/pod/delete.go +++ b/exec/pod/delete.go @@ -80,16 +80,14 @@ func (d *DeletePodActionExecutor) Exec(uid string, ctx context.Context, model *s } func (d *DeletePodActionExecutor) create(ctx context.Context, expModel *spec.ExpModel) *spec.Response { - podObjectMetaValues := ctx.Value(model.PodObjectMetaKey) - if podObjectMetaValues == nil { - errMsg := "less pod object meta parameter" - return spec.ReturnFailWitResult(spec.Code[spec.IllegalParameters], errMsg, - v1alpha1.CreateFailExperimentStatus(errMsg, nil)) + podObjectMetaList, err := model.ExtractPodObjectMetasFromContext(ctx) + if err != nil { + return spec.ReturnFailWitResult(spec.Code[spec.IllegalParameters], err.Error(), + v1alpha1.CreateFailExperimentStatus(err.Error(), nil)) } - podObjectMetas := podObjectMetaValues.([]model.PodObjectMeta) statuses := make([]v1alpha1.ResourceStatus, 0) success := false - for _, meta := range podObjectMetas { + for _, meta := range podObjectMetaList { status := v1alpha1.ResourceStatus{ Uid: meta.Uid, Name: meta.Name, diff --git a/exec/pod/osexp.go b/exec/pod/osexp.go index ee0c0cd..da3f597 100644 --- a/exec/pod/osexp.go +++ b/exec/pod/osexp.go @@ -19,10 +19,11 @@ package pod import ( "context" "fmt" - "strings" "github.com/chaosblade-io/chaosblade-exec-os/exec" "github.com/chaosblade-io/chaosblade-spec-go/spec" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" "github.com/chaosblade-io/chaosblade-operator/channel" "github.com/chaosblade-io/chaosblade-operator/exec/model" @@ -58,27 +59,56 @@ func NewOSSubResourceExecutor(client *channel.Client) spec.Executor { resourceIdentifier *model.ResourceIdentifier) ([]model.ExperimentIdentifier, error) { bladeBin := meta.Constant.BladeBin identifiers := make([]model.ExperimentIdentifier, 0) - - if expIdValues, ok := spec.IsDestroy(ctx); ok { - expIds := strings.Split(expIdValues, ",") - for _, expId := range expIds { - command := fmt.Sprintf("%s destroy %s", bladeBin, expId) - identifier := model.NewExperimentIdentifier(expId, resourceIdentifier.PodUid, - resourceIdentifier.PodName, command) + // Destroy + if _, ok := spec.IsDestroy(ctx); ok { + expObjectMetasMaps, err := model.ExtractNodeNameExpObjectMetasMapFromContext(ctx) + if err != nil { + return identifiers, err + } + expObjectMetas := expObjectMetasMaps[resourceIdentifier.NodeName] + for _, expObjectMeta := range expObjectMetas { + if expObjectMeta.Id == "" { + continue + } + command := fmt.Sprintf("%s destroy %s", bladeBin, expObjectMeta.Id) + identifier := model.NewExperimentIdentifier(expObjectMeta.Id, expObjectMeta.Uid, + expObjectMeta.Name, command) identifiers = append(identifiers, identifier) } return identifiers, nil } - + // Create matchers := spec.ConvertExpMatchersToString(expModel, model.ExcludeKeyFunc()) - containerId := resourceIdentifier.ContainerId - if containerId == "" { - return identifiers, fmt.Errorf("cannot find a valiable container in the %s pod", resourceIdentifier.PodName) + // Get pods from context + podObjectMetaList, err := model.ExtractPodObjectMetasFromContext(ctx) + if err != nil { + return identifiers, err + } + // Traverse the pod list to get the container running in every pod + for _, podObjectMeta := range podObjectMetaList { + if podObjectMeta.NodeName != resourceIdentifier.NodeName { + continue + } + pod := v1.Pod{} + err := client.Get(context.Background(), types.NamespacedName{ + Name: podObjectMeta.Name, + Namespace: podObjectMeta.Namespace}, &pod) + if err != nil { + identifier := model.NewExperimentIdentifierWithError("", podObjectMeta.Uid, podObjectMeta.Name, err.Error()) + identifiers = append(identifiers, identifier) + continue + } + containerId, err := model.GetOneAvailableContainerIdFromPod(pod) + if err != nil { + identifier := model.NewExperimentIdentifierWithError("", podObjectMeta.Uid, podObjectMeta.Name, err.Error()) + identifiers = append(identifiers, identifier) + continue + } + flags := fmt.Sprintf("%s --container-id %s", matchers, containerId) + command := fmt.Sprintf("%s create docker %s %s %s", bladeBin, expModel.Target, expModel.ActionName, flags) + identifier := model.NewExperimentIdentifier("", podObjectMeta.Uid, podObjectMeta.Name, command) + identifiers = append(identifiers, identifier) } - flags := fmt.Sprintf("%s --container-id %s", matchers, containerId) - command := fmt.Sprintf("%s create docker %s %s %s", bladeBin, expModel.Target, expModel.ActionName, flags) - identifier := model.NewExperimentIdentifier("", resourceIdentifier.PodUid, resourceIdentifier.PodName, command) - identifiers = append(identifiers, identifier) return identifiers, nil }, },