diff --git a/KubeArmor/common/common.go b/KubeArmor/common/common.go index 2249159c01..58d5a04741 100644 --- a/KubeArmor/common/common.go +++ b/KubeArmor/common/common.go @@ -20,6 +20,22 @@ import ( kg "github.com/kubearmor/KubeArmor/KubeArmor/log" ) +// ContainerRuntimeSocketMap lists all sockets that are auto-detected for each +// supported runtime +var ContainerRuntimeSocketMap = map[string][]string{ + "docker": { + "/var/run/docker.sock", + }, + "containerd": { + "/var/snap/microk8s/common/run/containerd.sock", + "/run/k3s/containerd/containerd.sock", + "/var/run/containerd/containerd.sock", + }, + "crio": { + "/var/run/crio/crio.sock", + }, +} + // ============ // // == Common == // // ============ // @@ -365,6 +381,17 @@ func IsK8sEnv() bool { return false } +// GetCRISocket Function validates and returns the socket for each compatible +// runtime +func GetCRISocket(ContainerRuntime string) string { + for _, candidate := range ContainerRuntimeSocketMap[ContainerRuntime] { + if _, err := os.Stat(candidate); err == nil { + return candidate + } + } + return "" +} + // ==================== // // == Identity Match == // // ==================== // diff --git a/KubeArmor/config/config.go b/KubeArmor/config/config.go index 533d090390..5e5d212183 100644 --- a/KubeArmor/config/config.go +++ b/KubeArmor/config/config.go @@ -4,6 +4,7 @@ package config import ( + "errors" "fmt" "os" "strings" @@ -40,6 +41,8 @@ type KubearmorConfig struct { HostDefaultCapabilitiesPosture string // Default Enforcement Action in Global Capabilities Context CoverageTest bool // Enable/Disable Coverage Test + + CRISocket string // The container runtime endpoint to use } // PolicyDir policy dir path for host policies backup @@ -105,6 +108,9 @@ const ConfigCoverageTest string = "coverageTest" // ConfigK8sEnv VM key const ConfigK8sEnv string = "k8s" +// ConfigCRISocket key +const ConfigCRISocket string = "criSocket" + func readCmdLineParams() { hostname, _ := os.Hostname() clusterStr := flag.String(ConfigCluster, "default", "cluster name") @@ -132,6 +138,8 @@ func readCmdLineParams() { coverageTestB := flag.Bool(ConfigCoverageTest, false, "enabling CoverageTest") + criSocket := flag.String(ConfigCRISocket, "", "path to CRI socket. Format: unix:///path/to/file.sock. If empty kubearmor will try to auto-detect this.") + flags := []string{} flag.VisitAll(func(f *flag.Flag) { kv := fmt.Sprintf("%s:%v", f.Name, f.Value) @@ -165,6 +173,8 @@ func readCmdLineParams() { viper.SetDefault(ConfigHostDefaultCapabilitiesPosture, *hostDefaultCapabilitiesPosture) viper.SetDefault(ConfigCoverageTest, *coverageTestB) + + viper.SetDefault(ConfigCRISocket, *criSocket) } // LoadConfig Load configuration @@ -213,6 +223,16 @@ func LoadConfig() error { GlobalCfg.HostDefaultNetworkPosture = viper.GetString(ConfigHostDefaultNetworkPosture) GlobalCfg.HostDefaultCapabilitiesPosture = viper.GetString(ConfigHostDefaultCapabilitiesPosture) + // read CRI_SOCKET env variable. If empty, check criSocket flag + GlobalCfg.CRISocket = os.Getenv("CRI_SOCKET") + if GlobalCfg.CRISocket == "" { + GlobalCfg.CRISocket = viper.GetString(ConfigCRISocket) + } + + if GlobalCfg.CRISocket != "" && !strings.HasPrefix(GlobalCfg.CRISocket, "unix://") { + return errors.New(fmt.Sprintf("%s is invalid. CRI socket must start with unix://", GlobalCfg.CRISocket)) + } + kg.Printf("Configuration [%+v]", GlobalCfg) if GlobalCfg.KVMAgent { diff --git a/KubeArmor/core/containerdHandler.go b/KubeArmor/core/containerdHandler.go index dd9b7a894a..1e7defb231 100644 --- a/KubeArmor/core/containerdHandler.go +++ b/KubeArmor/core/containerdHandler.go @@ -7,8 +7,8 @@ import ( "context" "fmt" "os" - "path/filepath" "strconv" + "strings" "time" kl "github.com/kubearmor/KubeArmor/KubeArmor/common" @@ -42,8 +42,6 @@ func init() { typeurl.Register(&specs.Spec{}, prefix, "opencontainers/runtime-spec", major, "Spec") typeurl.Register(&specs.Process{}, prefix, "opencontainers/runtime-spec", major, "Process") - - Containerd = NewContainerdHandler() } // ContainerdHandler Structure @@ -72,29 +70,15 @@ type ContainerdHandler struct { func NewContainerdHandler() *ContainerdHandler { ch := &ContainerdHandler{} - sockFile := "unix://" - - for idx, candidate := range []string{"/var/run/containerd/containerd.sock", "/var/snap/microk8s/common/run/containerd.sock", "/run/k3s/containerd/containerd.sock"} { - if _, err := os.Stat(filepath.Clean(candidate)); err == nil { - sockFile = sockFile + candidate - - if idx == 0 { // containerd - ch.StoragePath = "/run/containerd" - } else if idx == 1 { // microk8s - ch.StoragePath = "/var/snap/microk8s/common/run/containerd" - } else if idx == 2 { // k3s - ch.StoragePath = "/run/k3s/containerd" - } - - break - } - } - - if sockFile == "unix://" { - return nil + if strings.Contains(cfg.GlobalCfg.CRISocket, "microk8s") { // microk8s + ch.StoragePath = "/var/snap/microk8s/common/run/containerd" + } else if strings.Contains(cfg.GlobalCfg.CRISocket, "k3s") { // k3s + ch.StoragePath = "/run/k3s/containerd" + } else { // vanilla containerd + ch.StoragePath = "/run/containerd" } - conn, err := grpc.Dial(sockFile, grpc.WithInsecure()) + conn, err := grpc.Dial(cfg.GlobalCfg.CRISocket, grpc.WithInsecure()) if err != nil { return nil } @@ -380,6 +364,8 @@ func (dm *KubeArmorDaemon) UpdateContainerdContainer(ctx context.Context, contai // MonitorContainerdEvents Function func (dm *KubeArmorDaemon) MonitorContainerdEvents() { + Containerd = NewContainerdHandler() + // check if Containerd exists if Containerd == nil { return diff --git a/KubeArmor/core/crioHandler.go b/KubeArmor/core/crioHandler.go index b54c2ca965..3eb891a5f7 100644 --- a/KubeArmor/core/crioHandler.go +++ b/KubeArmor/core/crioHandler.go @@ -8,7 +8,6 @@ import ( "encoding/json" "fmt" "os" - "path/filepath" "strconv" "time" @@ -29,9 +28,6 @@ type CrioHandler struct { // crio client client pb.RuntimeServiceClient - // storage path - StoragePath string - // containers is a map with empty value to have lookups in constant time containers map[string]struct{} } @@ -48,36 +44,11 @@ type CrioContainerInfo struct { // Crio Handler var Crio *CrioHandler -// init Function -func init() { - Crio = NewCrioHandler() -} - // NewCrioHandler Function creates a new Crio handler func NewCrioHandler() *CrioHandler { ch := &CrioHandler{} - sockFile := "unix://" - - // TODO: identify other k8s distros which support CRIO by default - for idx, candidate := range []string{"/var/run/crio/crio.sock"} { - if _, err := os.Stat(filepath.Clean(candidate)); err == nil { - sockFile = sockFile + candidate - - if idx == 0 { - // default crio - ch.StoragePath = "/run/crio" - } - - break - } - } - - if sockFile == "unix://" { - return nil - } - - conn, err := grpc.Dial(sockFile, grpc.WithInsecure()) + conn, err := grpc.Dial(cfg.GlobalCfg.CRISocket, grpc.WithInsecure()) if err != nil { return nil } @@ -339,6 +310,7 @@ func (dm *KubeArmorDaemon) UpdateCrioContainer(ctx context.Context, containerID, // MonitorCrioEvents Function func (dm *KubeArmorDaemon) MonitorCrioEvents() { + Crio = NewCrioHandler() // check if Crio exists if Crio == nil { return diff --git a/KubeArmor/core/dockerHandler.go b/KubeArmor/core/dockerHandler.go index ef67d74ba1..761ca1e855 100644 --- a/KubeArmor/core/dockerHandler.go +++ b/KubeArmor/core/dockerHandler.go @@ -29,11 +29,6 @@ import ( // Docker Handler var Docker *DockerHandler -// init Function -func init() { - Docker = NewDockerHandler() -} - // DockerVersion Structure type DockerVersion struct { APIVersion string `json:"ApiVersion"` @@ -52,7 +47,7 @@ func NewDockerHandler() *DockerHandler { // specify the docker api version that we want to use // Versioned API: https://docs.docker.com/engine/api/ - versionStr, err := kl.GetCommandOutputWithErr("curl", []string{"--silent", "--unix-socket", "/var/run/docker.sock", "http://localhost/version"}) + versionStr, err := kl.GetCommandOutputWithErr("curl", []string{"--silent", "--unix-socket", strings.TrimPrefix(cfg.GlobalCfg.CRISocket, "unix://"), "http://localhost/version"}) if err != nil { return nil } @@ -381,6 +376,8 @@ func (dm *KubeArmorDaemon) MonitorDockerEvents() { dm.WgDaemon.Add(1) defer dm.WgDaemon.Done() + Docker = NewDockerHandler() + // check if Docker exists if Docker == nil { return diff --git a/KubeArmor/core/kubeArmor.go b/KubeArmor/core/kubeArmor.go index 6ab245306d..34bd3299c3 100644 --- a/KubeArmor/core/kubeArmor.go +++ b/KubeArmor/core/kubeArmor.go @@ -11,6 +11,7 @@ import ( "syscall" "time" + "github.com/kubearmor/KubeArmor/KubeArmor/common" kl "github.com/kubearmor/KubeArmor/KubeArmor/common" cfg "github.com/kubearmor/KubeArmor/KubeArmor/config" kg "github.com/kubearmor/KubeArmor/KubeArmor/log" @@ -449,84 +450,104 @@ func KubeArmor() { // == // if dm.K8sEnabled && cfg.GlobalCfg.Policy { - if strings.HasPrefix(dm.Node.ContainerRuntimeVersion, "docker") { - sockFile := false + // check if the CRI socket set while executing kubearmor exists + if cfg.GlobalCfg.CRISocket != "" { + trimmedSocket := strings.TrimPrefix(cfg.GlobalCfg.CRISocket, "unix://") + if _, err := os.Stat(trimmedSocket); err != nil { + dm.Logger.Warnf("Error while looking for CRI socket file: %s", err.Error()) - for _, candidate := range []string{"/var/run/docker.sock"} { - if _, err := os.Stat(candidate); err == nil { - sockFile = true - break - } + // destroy the daemon + dm.DestroyKubeArmorDaemon() + return } - if sockFile { + // monitor containers + if strings.Contains(cfg.GlobalCfg.CRISocket, "docker") { // update already deployed containers dm.GetAlreadyDeployedDockerContainers() - // monitor docker events go dm.MonitorDockerEvents() + } else if strings.Contains(cfg.GlobalCfg.CRISocket, "crio") { + // monitor crio events + go dm.MonitorCrioEvents() + } else if strings.Contains(cfg.GlobalCfg.CRISocket, "containerd") { + // monitor containerd events + go dm.MonitorContainerdEvents() } else { - for _, candidate := range []string{"/var/run/containerd/containerd.sock"} { - if _, err := os.Stat(candidate); err == nil { - sockFile = true - break + dm.Logger.Errf("Failed to monitor containers: %s is not a supported CRI socket.", cfg.GlobalCfg.CRISocket) + // destroy the daemon + dm.DestroyKubeArmorDaemon() + return + } + + dm.Logger.Printf("Using %s for monitoring containers.", cfg.GlobalCfg.CRISocket) + + } else { // CRI socket not set, we'll have to auto detect + dm.Logger.Print("CRI socket not set. Trying to detect.") + + if strings.HasPrefix(dm.Node.ContainerRuntimeVersion, "docker") { + socketFile := common.GetCRISocket("docker") + + if socketFile != "" { + cfg.GlobalCfg.CRISocket = "unix://" + socketFile + + // update already deployed containers + dm.GetAlreadyDeployedDockerContainers() + + // monitor docker events + go dm.MonitorDockerEvents() + } else { + // we might have to use containerd's socket as docker's socket is not + // available + socketFile := common.GetCRISocket("containerd") + + if socketFile != "" { + cfg.GlobalCfg.CRISocket = "unix://" + socketFile + + // monitor containerd events + go dm.MonitorContainerdEvents() + } else { + dm.Logger.Err("Failed to monitor containers (Docker socket file is not accessible)") + + // destroy the daemon + dm.DestroyKubeArmorDaemon() + + return } } + } else if strings.HasPrefix(dm.Node.ContainerRuntimeVersion, "cri-o") { // cri-o + socketFile := common.GetCRISocket("crio") - if sockFile { - // monitor containerd events - go dm.MonitorContainerdEvents() + if socketFile != "" { + cfg.GlobalCfg.CRISocket = "unix://" + socketFile + + // monitor cri-o events + go dm.MonitorCrioEvents() } else { - dm.Logger.Err("Failed to monitor containers (Docker socket file is not accessible)") + dm.Logger.Err("Failed to monitor containers (CRI-O socket file is not accessible)") // destroy the daemon dm.DestroyKubeArmorDaemon() return } - } - } else if strings.HasPrefix(dm.Node.ContainerRuntimeVersion, "cri-o") { // cri-o - sockFile := false + } else { // containerd + socketFile := common.GetCRISocket("containerd") - for _, candidate := range []string{"/var/run/crio/crio.sock"} { - if _, err := os.Stat(candidate); err == nil { - sockFile = true - break - } - } + if socketFile != "" { + cfg.GlobalCfg.CRISocket = "unix://" + socketFile - if sockFile { - // monitor cri-o events - go dm.MonitorCrioEvents() - } else { - dm.Logger.Err("Failed to monitor containers (CRI-O socket file is not accessible)") - - // destroy the daemon - dm.DestroyKubeArmorDaemon() + // monitor containerd events + go dm.MonitorContainerdEvents() + } else { + dm.Logger.Err("Failed to monitor containers (Containerd socket file is not accessible)") - return - } - } else { // containerd - sockFile := false + // destroy the daemon + dm.DestroyKubeArmorDaemon() - for _, candidate := range []string{"/var/run/containerd/containerd.sock", "/var/snap/microk8s/common/run/containerd.sock", "/run/k3s/containerd/containerd.sock"} { - if _, err := os.Stat(candidate); err == nil { - sockFile = true - break + return } } - - if sockFile { - // monitor containerd events - go dm.MonitorContainerdEvents() - } else { - dm.Logger.Err("Failed to monitor containers (Containerd socket file is not accessible)") - - // destroy the daemon - dm.DestroyKubeArmorDaemon() - - return - } } }