From c065c7fa5cf9995b78d659c2427c0eb9a3247372 Mon Sep 17 00:00:00 2001 From: hasbro17 Date: Wed, 28 Jun 2017 15:57:40 -0700 Subject: [PATCH] *: audit operator actions to hostPath volume --- pkg/cluster/audit.go | 98 ++++++++++++++++++++++++++++++++++++ pkg/cluster/cluster.go | 20 +++++--- pkg/cluster/self_hosted.go | 3 ++ pkg/util/k8sutil/pod_util.go | 9 ++++ 4 files changed, 123 insertions(+), 7 deletions(-) create mode 100644 pkg/cluster/audit.go diff --git a/pkg/cluster/audit.go b/pkg/cluster/audit.go new file mode 100644 index 000000000..856b4e102 --- /dev/null +++ b/pkg/cluster/audit.go @@ -0,0 +1,98 @@ +// Copyright 2017 The etcd-operator Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cluster + +import ( + "os" + + "github.com/coreos/etcd-operator/pkg/spec" + "github.com/coreos/etcd-operator/pkg/util/k8sutil" + + "github.com/Sirupsen/logrus" + "k8s.io/client-go/pkg/api/v1" +) + +func NewAuditLogger(cl *spec.Cluster, lg *logrus.Entry) *logrus.Logger { + if cl.Spec.SelfHosted == nil { + return nil + } + + mountPath := "/var/tmp/etcd-operator/" + _, err := os.Stat(mountPath) + if os.IsNotExist(err) { + lg.Infof("mountPath(%v) not detected, no auditing will be performed: %v", mountPath, err) + return nil + } + lg.Infof("detected the mountPath(%v): starting to audit operator actions to the mountPath", mountPath) + + fileName := mountPath + cl.Metadata.Name + ".log" + logFile, err := os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + lg.Errorf("failed to open logfile(%v): %v", fileName, err) + return nil + } + + l := logrus.New() + l.Out = logFile + l.Infof("Starting audit logs for self-hosted etcd cluster: %v", cl.Metadata.Name) + return l +} + +func (c *Cluster) auditPodCreation(pod *v1.Pod, podCreationErr error) { + if c.loggerSH == nil { + return + } + + if podCreationErr != nil { + c.loggerSH.Infof("failed to create pod (%s): %v ", pod.Name, podCreationErr) + return + } + + podSpec, err := k8sutil.GetReadablePodSpec(pod) + if err != nil { + c.loggerSH.Infof("failed to get readable spec for pod(%v): %v ", pod.Name, err) + } + c.loggerSH.Infof("created pod (%s) with spec: %s\n", pod.Name, podSpec) +} + +func (c *Cluster) auditPodDeletion(podName string, podDeletionErr error) { + if c.loggerSH == nil { + return + } + + if podDeletionErr != nil { + if !k8sutil.IsKubernetesResourceNotFoundError(podDeletionErr) { + c.loggerSH.Infof("failed to delete pod (%s): %v ", podName, podDeletionErr) + return + } + c.loggerSH.Infof("pod (%s) not found while trying to delete: %v ", podName) + return + } + c.loggerSH.Infof("deleted pod (%s)", podName) +} + +func (c *Cluster) auditClusterSpecUpdate(oldSpec, newSpec string) { + if c.loggerSH == nil { + return + } + c.loggerSH.Infof("spec update: \nOld:\n%v \nNew:\n%v\n", oldSpec, newSpec) +} + +func (c *Cluster) auditMessage(msg string) { + if c.loggerSH == nil { + return + } + c.loggerSH.Infof(msg) +} diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index ee6669b87..dc68d9e4e 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -65,6 +65,8 @@ type Config struct { type Cluster struct { logger *logrus.Entry + // file logger for self hosted cluster + loggerSH *logrus.Logger config Config @@ -93,13 +95,14 @@ type Cluster struct { func New(config Config, cl *spec.Cluster, stopC <-chan struct{}, wg *sync.WaitGroup) *Cluster { lg := logrus.WithField("pkg", "cluster").WithField("cluster-name", cl.Metadata.Name) c := &Cluster{ - logger: lg, - config: config, - cluster: cl, - eventCh: make(chan *clusterEvent, 100), - stopCh: make(chan struct{}), - status: cl.Status.Copy(), - gc: garbagecollection.New(config.KubeCli, cl.Metadata.Namespace), + logger: lg, + loggerSH: NewAuditLogger(cl, lg), + config: config, + cluster: cl, + eventCh: make(chan *clusterEvent, 100), + stopCh: make(chan struct{}), + status: cl.Status.Copy(), + gc: garbagecollection.New(config.KubeCli, cl.Metadata.Namespace), } wg.Add(1) @@ -246,6 +249,7 @@ func (c *Cluster) run(stopC <-chan struct{}) { c.reportFailedStatus() c.logger.Infof("deleting the failed cluster") + c.auditMessage("deleting the failed cluster") c.delete() } @@ -476,6 +480,7 @@ func (c *Cluster) removePod(name string) error { ns := c.cluster.Metadata.Namespace opts := metav1.NewDeleteOptions(podTerminationGracePeriod) err := c.config.KubeCli.Core().Pods(ns).Delete(name, opts) + c.auditPodDeletion(name, err) if err != nil { if !k8sutil.IsKubernetesResourceNotFoundError(err) { return err @@ -618,4 +623,5 @@ func (c *Cluster) logSpecUpdate(newSpec spec.ClusterSpec) { c.logger.Errorf("failed to marshal cluster spec: %v", err) } c.logger.Infof("spec update: \nOld:\n%v \nNew:\n%v", string(oldSpecBytes), string(newSpecBytes)) + c.auditClusterSpecUpdate(string(oldSpecBytes), string(newSpecBytes)) } diff --git a/pkg/cluster/self_hosted.go b/pkg/cluster/self_hosted.go index 15a85a258..3451fda6b 100644 --- a/pkg/cluster/self_hosted.go +++ b/pkg/cluster/self_hosted.go @@ -132,6 +132,7 @@ func (c *Cluster) addOneSelfHostedMember() error { pod := k8sutil.NewSelfHostedEtcdPod(newMember, initialCluster, c.members.ClientURLs(), c.cluster.Metadata.Name, "existing", "", c.cluster.Spec, c.cluster.AsOwner()) _, err = c.config.KubeCli.CoreV1().Pods(ns).Create(pod) + c.auditPodCreation(pod, err) if err != nil { return err } @@ -155,6 +156,7 @@ func (c *Cluster) newSelfHostedSeedMember() error { pod := k8sutil.NewSelfHostedEtcdPod(newMember, initialCluster, nil, c.cluster.Metadata.Name, "new", uuid.New(), c.cluster.Spec, c.cluster.AsOwner()) _, err := k8sutil.CreateAndWaitPod(c.config.KubeCli, c.cluster.Metadata.Namespace, pod, 30*time.Second) + c.auditPodCreation(pod, err) if err != nil { return err } @@ -192,6 +194,7 @@ func (c *Cluster) migrateBootMember() error { pod := k8sutil.NewSelfHostedEtcdPod(newMember, initialCluster, []string{endpoint}, c.cluster.Metadata.Name, "existing", "", c.cluster.Spec, c.cluster.AsOwner()) ns := c.cluster.Metadata.Namespace _, err = k8sutil.CreateAndWaitPod(c.config.KubeCli, ns, pod, 30*time.Second) + c.auditPodCreation(pod, err) if err != nil { return err } diff --git a/pkg/util/k8sutil/pod_util.go b/pkg/util/k8sutil/pod_util.go index 86098cfff..de497af1b 100644 --- a/pkg/util/k8sutil/pod_util.go +++ b/pkg/util/k8sutil/pod_util.go @@ -15,6 +15,7 @@ package k8sutil import ( + "encoding/json" "fmt" "github.com/coreos/etcd-operator/pkg/spec" @@ -168,3 +169,11 @@ func getPodReadyCondition(status *v1.PodStatus) *v1.PodCondition { } return nil } + +func GetReadablePodSpec(pod *v1.Pod) (string, error) { + bytes, err := json.MarshalIndent(pod.Spec, "", " ") + if err != nil { + return "", err + } + return string(bytes), nil +}