*: audit operator actions to hostPath volume

pkg/cluster/audit.go

@@ -0,0 +1,98 @@
+// Copyright 2017 The etcd-operator Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cluster
+
+import (
+	"os"
+
+	"github.com/coreos/etcd-operator/pkg/spec"
+	"github.com/coreos/etcd-operator/pkg/util/k8sutil"
+
+	"github.com/Sirupsen/logrus"
+	"k8s.io/client-go/pkg/api/v1"
+)
+
+func NewAuditLogger(cl *spec.Cluster, lg *logrus.Entry) *logrus.Logger {
+	if cl.Spec.SelfHosted == nil {
+		return nil
+	}
+
+	mountPath := "/var/tmp/etcd-operator/"
+	_, err := os.Stat(mountPath)
+	if os.IsNotExist(err) {
+		lg.Infof("mountPath(%v) not detected, no auditing will be performed: %v", mountPath, err)
+		return nil
+	}
+	lg.Infof("detected the mountPath(%v): starting to audit operator actions to the mountPath", mountPath)
+
+	fileName := mountPath + cl.Metadata.Name + ".log"
+	logFile, err := os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
+	if err != nil {
+		lg.Errorf("failed to open logfile(%v): %v", fileName, err)
+		return nil
+	}
+
+	l := logrus.New()
+	l.Out = logFile
+	l.Infof("Starting audit logs for self-hosted etcd cluster: %v", cl.Metadata.Name)
+	return l
+}
+
+func (c *Cluster) auditPodCreation(pod *v1.Pod, podCreationErr error) {
+	if c.loggerSH == nil {
+		return
+	}
+
+	if podCreationErr != nil {
+		c.loggerSH.Infof("failed to create pod (%s): %v ", pod.Name, podCreationErr)
+		return
+	}
+
+	podSpec, err := k8sutil.GetReadablePodSpec(pod)
+	if err != nil {
+		c.loggerSH.Infof("failed to get readable spec for pod(%v): %v ", pod.Name, err)
+	}
+	c.loggerSH.Infof("created pod (%s) with spec: %s\n", pod.Name, podSpec)
+}
+
+func (c *Cluster) auditPodDeletion(podName string, podDeletionErr error) {
+	if c.loggerSH == nil {
+		return
+	}
+
+	if podDeletionErr != nil {
+		if !k8sutil.IsKubernetesResourceNotFoundError(podDeletionErr) {
+			c.loggerSH.Infof("failed to delete pod (%s): %v ", podName, podDeletionErr)
+			return
+		}
+		c.loggerSH.Infof("pod (%s) not found while trying to delete: %v ", podName)
+		return
+	}
+	c.loggerSH.Infof("deleted pod (%s)", podName)
+}
+
+func (c *Cluster) auditClusterSpecUpdate(oldSpec, newSpec string) {
+	if c.loggerSH == nil {
+		return
+	}
+	c.loggerSH.Infof("spec update: \nOld:\n%v \nNew:\n%v\n", oldSpec, newSpec)
+}
+
+func (c *Cluster) auditMessage(msg string) {
+	if c.loggerSH == nil {
+		return
+	}
+	c.loggerSH.Infof(msg)
+}

pkg/cluster/cluster.go

@@ -65,6 +65,8 @@ type Config struct {
 
 type Cluster struct {
 	logger *logrus.Entry
+	// file logger for self hosted cluster
+	loggerSH *logrus.Logger
 
 	config Config
 
@@ -93,13 +95,14 @@ type Cluster struct {
 func New(config Config, cl *spec.Cluster, stopC <-chan struct{}, wg *sync.WaitGroup) *Cluster {
 	lg := logrus.WithField("pkg", "cluster").WithField("cluster-name", cl.Metadata.Name)
 	c := &Cluster{
-		logger:  lg,
-		config:  config,
-		cluster: cl,
-		eventCh: make(chan *clusterEvent, 100),
-		stopCh:  make(chan struct{}),
-		status:  cl.Status.Copy(),
-		gc:      garbagecollection.New(config.KubeCli, cl.Metadata.Namespace),
+		logger:   lg,
+		loggerSH: NewAuditLogger(cl, lg),
+		config:   config,
+		cluster:  cl,
+		eventCh:  make(chan *clusterEvent, 100),
+		stopCh:   make(chan struct{}),
+		status:   cl.Status.Copy(),
+		gc:       garbagecollection.New(config.KubeCli, cl.Metadata.Namespace),
 	}
 
 	wg.Add(1)
@@ -246,6 +249,7 @@ func (c *Cluster) run(stopC <-chan struct{}) {
 			c.reportFailedStatus()
 
 			c.logger.Infof("deleting the failed cluster")
+			c.auditMessage("deleting the failed cluster")
 			c.delete()
 		}
 
@@ -476,6 +480,7 @@ func (c *Cluster) removePod(name string) error {
 	ns := c.cluster.Metadata.Namespace
 	opts := metav1.NewDeleteOptions(podTerminationGracePeriod)
 	err := c.config.KubeCli.Core().Pods(ns).Delete(name, opts)
+	c.auditPodDeletion(name, err)
 	if err != nil {
 		if !k8sutil.IsKubernetesResourceNotFoundError(err) {
 			return err
@@ -618,4 +623,5 @@ func (c *Cluster) logSpecUpdate(newSpec spec.ClusterSpec) {
 		c.logger.Errorf("failed to marshal cluster spec: %v", err)
 	}
 	c.logger.Infof("spec update: \nOld:\n%v \nNew:\n%v", string(oldSpecBytes), string(newSpecBytes))
+	c.auditClusterSpecUpdate(string(oldSpecBytes), string(newSpecBytes))
 }

pkg/cluster/self_hosted.go

@@ -132,6 +132,7 @@ func (c *Cluster) addOneSelfHostedMember() error {
 	pod := k8sutil.NewSelfHostedEtcdPod(newMember, initialCluster, c.members.ClientURLs(), c.cluster.Metadata.Name, "existing", "", c.cluster.Spec, c.cluster.AsOwner())
 
 	_, err = c.config.KubeCli.CoreV1().Pods(ns).Create(pod)
+	c.auditPodCreation(pod, err)
 	if err != nil {
 		return err
 	}
@@ -155,6 +156,7 @@ func (c *Cluster) newSelfHostedSeedMember() error {
 
 	pod := k8sutil.NewSelfHostedEtcdPod(newMember, initialCluster, nil, c.cluster.Metadata.Name, "new", uuid.New(), c.cluster.Spec, c.cluster.AsOwner())
 	_, err := k8sutil.CreateAndWaitPod(c.config.KubeCli, c.cluster.Metadata.Namespace, pod, 30*time.Second)
+	c.auditPodCreation(pod, err)
 	if err != nil {
 		return err
 	}
@@ -192,6 +194,7 @@ func (c *Cluster) migrateBootMember() error {
 	pod := k8sutil.NewSelfHostedEtcdPod(newMember, initialCluster, []string{endpoint}, c.cluster.Metadata.Name, "existing", "", c.cluster.Spec, c.cluster.AsOwner())
 	ns := c.cluster.Metadata.Namespace
 	_, err = k8sutil.CreateAndWaitPod(c.config.KubeCli, ns, pod, 30*time.Second)
+	c.auditPodCreation(pod, err)
 	if err != nil {
 		return err
 	}

pkg/util/k8sutil/pod_util.go

@@ -15,6 +15,7 @@
 package k8sutil
 
 import (
+	"encoding/json"
 	"fmt"
 
 	"github.com/coreos/etcd-operator/pkg/spec"
@@ -168,3 +169,11 @@ func getPodReadyCondition(status *v1.PodStatus) *v1.PodCondition {
 	}
 	return nil
 }
+
+func GetReadablePodSpec(pod *v1.Pod) (string, error) {
+	bytes, err := json.MarshalIndent(pod.Spec, "", "    ")
+	if err != nil {
+		return "", err
+	}
+	return string(bytes), nil
+}