diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 79da869f47..872130b107 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -54,7 +54,7 @@ jobs: - name: Setup Enviroment run: | - ./contribution/k3s/install_k3s.sh + RUNTIME=docker ./contribution/k3s/install_k3s.sh - name: Install annotation controller run: | diff --git a/KubeArmor/build/kubearmor-test-crio.yaml b/KubeArmor/build/kubearmor-test-crio.yaml new file mode 100644 index 0000000000..a396dc6dea --- /dev/null +++ b/KubeArmor/build/kubearmor-test-crio.yaml @@ -0,0 +1,1001 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubearmor + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubearmor + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: kubearmor + namespace: kube-system +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + kubearmor-app: kubearmor + name: kubearmor + namespace: kube-system +spec: + selector: + matchLabels: + kubearmor-app: kubearmor + template: + metadata: + annotations: + container.apparmor.security.beta.kubernetes.io/kubearmor: unconfined + labels: + kubearmor-app: kubearmor + spec: + containers: + - args: + - -gRPC=32767 + - -logPath=/tmp/kubearmor.log + - -enableKubeArmorHostPolicy + image: kubearmor/kubearmor:latest + imagePullPolicy: Never + livenessProbe: + exec: + command: + - /bin/bash + - -c + - if [ -z $(pgrep kubearmor) ]; then exit 1; fi; + initialDelaySeconds: 60 + periodSeconds: 10 + name: kubearmor + ports: + - containerPort: 32767 + securityContext: + privileged: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /lib/modules + name: lib-modules-path + readOnly: true + - mountPath: /sys/fs/bpf + name: sys-fs-bpf-path + - mountPath: /sys/kernel/security + name: sys-kernel-security-path + - mountPath: /sys/kernel/debug + name: sys-kernel-debug-path + - mountPath: /media/root/etc/os-release + name: os-release-path + readOnly: true + - mountPath: /usr/src + name: usr-src-path + readOnly: true + - mountPath: /etc/apparmor.d + name: etc-apparmor-d-path + - mountPath: /var/run/crio/crio.sock + name: crio-sock-path + readOnly: true + - mountPath: /run/crio + name: crio-storage-path + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kubernetes.io/os: linux + restartPolicy: Always + serviceAccountName: kubearmor + terminationGracePeriodSeconds: 30 + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /lib/modules + type: Directory + name: lib-modules-path + - hostPath: + path: /sys/fs/bpf + type: Directory + name: sys-fs-bpf-path + - hostPath: + path: /sys/kernel/security + type: Directory + name: sys-kernel-security-path + - hostPath: + path: /sys/kernel/debug + type: Directory + name: sys-kernel-debug-path + - hostPath: + path: /etc/os-release + type: File + name: os-release-path + - hostPath: + path: /usr/src + type: Directory + name: usr-src-path + - hostPath: + path: /etc/apparmor.d + type: DirectoryOrCreate + name: etc-apparmor-d-path + - hostPath: + path: /var/run/crio/crio.sock + type: Socket + name: crio-sock-path + - hostPath: + path: /run/crio + type: DirectoryOrCreate + name: crio-storage-path +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.1 + name: kubearmorpolicies.security.kubearmor.com +spec: + group: security.kubearmor.com + names: + kind: KubeArmorPolicy + listKind: KubeArmorPolicyList + plural: kubearmorpolicies + shortNames: + - ksp + singular: kubearmorpolicy + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: KubeArmorPolicy is the Schema for the kubearmorpolicies API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: KubeArmorPolicySpec defines the desired state of KubeArmorPolicy + properties: + action: + enum: + - Allow + - Audit + - Block + type: string + apparmor: + type: string + capabilities: + properties: + action: + enum: + - Allow + - Audit + - Block + type: string + matchCapabilities: + items: + properties: + action: + enum: + - Allow + - Audit + - Block + type: string + capability: + pattern: (chown|dac_override|dac_read_search|fowner|fsetid|kill|setgid|setuid|setpcap|linux_immutable|net_bind_service|net_broadcast|net_admin|net_raw|ipc_lock|ipc_owner|sys_module|sys_rawio|sys_chroot|sys_ptrace|sys_pacct|sys_admin|sys_boot|sys_nice|sys_resource|sys_time|sys_tty_config|mknod|lease|audit_write|audit_control|setfcap|mac_override|mac_admin)$ + type: string + fromSource: + items: + properties: + path: + pattern: ^\/+.*[^\/]$ + type: string + type: object + type: array + message: + type: string + severity: + maximum: 10 + minimum: 1 + type: integer + tags: + items: + type: string + type: array + required: + - capability + type: object + type: array + message: + type: string + severity: + maximum: 10 + minimum: 1 + type: integer + tags: + items: + type: string + type: array + required: + - matchCapabilities + type: object + file: + properties: + action: + enum: + - Allow + - Audit + - Block + type: string + matchDirectories: + items: + properties: + action: + enum: + - Allow + - Audit + - Block + type: string + dir: + pattern: ^\/$|^\/.*\/$ + type: string + fromSource: + items: + properties: + path: + pattern: ^\/+.*[^\/]$ + type: string + type: object + type: array + message: + type: string + ownerOnly: + type: boolean + readOnly: + type: boolean + recursive: + type: boolean + severity: + maximum: 10 + minimum: 1 + type: integer + tags: + items: + type: string + type: array + required: + - dir + type: object + type: array + matchPaths: + items: + properties: + action: + enum: + - Allow + - Audit + - Block + type: string + fromSource: + items: + properties: + path: + pattern: ^\/+.*[^\/]$ + type: string + type: object + type: array + message: + type: string + ownerOnly: + type: boolean + path: + pattern: ^\/+.*[^\/]$ + type: string + readOnly: + type: boolean + severity: + maximum: 10 + minimum: 1 + type: integer + tags: + items: + type: string + type: array + required: + - path + type: object + type: array + matchPatterns: + items: + properties: + action: + enum: + - Allow + - Audit + - Block + type: string + message: + type: string + ownerOnly: + type: boolean + pattern: + type: string + readOnly: + type: boolean + severity: + maximum: 10 + minimum: 1 + type: integer + tags: + items: + type: string + type: array + required: + - pattern + type: object + type: array + message: + type: string + severity: + maximum: 10 + minimum: 1 + type: integer + tags: + items: + type: string + type: array + type: object + message: + type: string + network: + properties: + action: + enum: + - Allow + - Audit + - Block + type: string + matchProtocols: + items: + properties: + action: + enum: + - Allow + - Audit + - Block + type: string + fromSource: + items: + properties: + path: + pattern: ^\/+.*[^\/]$ + type: string + type: object + type: array + message: + type: string + protocol: + pattern: (icmp|ICMP|tcp|TCP|udp|UDP|raw|RAW)$ + type: string + severity: + maximum: 10 + minimum: 1 + type: integer + tags: + items: + type: string + type: array + required: + - protocol + type: object + type: array + message: + type: string + severity: + maximum: 10 + minimum: 1 + type: integer + tags: + items: + type: string + type: array + required: + - matchProtocols + type: object + process: + properties: + action: + enum: + - Allow + - Audit + - Block + type: string + matchDirectories: + items: + properties: + action: + enum: + - Allow + - Audit + - Block + type: string + dir: + pattern: ^\/$|^\/.*\/$ + type: string + fromSource: + items: + properties: + path: + pattern: ^\/+.*[^\/]$ + type: string + type: object + type: array + message: + type: string + ownerOnly: + type: boolean + recursive: + type: boolean + severity: + maximum: 10 + minimum: 1 + type: integer + tags: + items: + type: string + type: array + required: + - dir + type: object + type: array + matchPaths: + items: + properties: + action: + enum: + - Allow + - Audit + - Block + type: string + fromSource: + items: + properties: + path: + pattern: ^\/+.*[^\/]$ + type: string + type: object + type: array + message: + type: string + ownerOnly: + type: boolean + path: + pattern: ^\/+.*[^\/]$ + type: string + severity: + maximum: 10 + minimum: 1 + type: integer + tags: + items: + type: string + type: array + required: + - path + type: object + type: array + matchPatterns: + items: + properties: + action: + enum: + - Allow + - Audit + - Block + type: string + message: + type: string + ownerOnly: + type: boolean + pattern: + type: string + severity: + maximum: 10 + minimum: 1 + type: integer + tags: + items: + type: string + type: array + required: + - pattern + type: object + type: array + message: + type: string + severity: + maximum: 10 + minimum: 1 + type: integer + tags: + items: + type: string + type: array + type: object + selector: + properties: + matchLabels: + additionalProperties: + type: string + type: object + type: object + severity: + maximum: 10 + minimum: 1 + type: integer + tags: + items: + type: string + type: array + required: + - selector + type: object + status: + description: KubeArmorPolicyStatus defines the observed state of KubeArmorPolicy + properties: + status: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.1 + name: kubearmorhostpolicies.security.kubearmor.com +spec: + group: security.kubearmor.com + names: + kind: KubeArmorHostPolicy + listKind: KubeArmorHostPolicyList + plural: kubearmorhostpolicies + shortNames: + - hsp + singular: kubearmorhostpolicy + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: KubeArmorHostPolicy is the Schema for the kubearmorhostpolicies + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: KubeArmorHostPolicySpec defines the desired state of KubeArmorHostPolicy + properties: + action: + enum: + - Allow + - Audit + - Block + type: string + apparmor: + type: string + capabilities: + properties: + action: + enum: + - Allow + - Audit + - Block + type: string + matchCapabilities: + items: + properties: + action: + enum: + - Allow + - Audit + - Block + type: string + capability: + pattern: (chown|dac_override|dac_read_search|fowner|fsetid|kill|setgid|setuid|setpcap|linux_immutable|net_bind_service|net_broadcast|net_admin|net_raw|ipc_lock|ipc_owner|sys_module|sys_rawio|sys_chroot|sys_ptrace|sys_pacct|sys_admin|sys_boot|sys_nice|sys_resource|sys_time|sys_tty_config|mknod|lease|audit_write|audit_control|setfcap|mac_override|mac_admin)$ + type: string + fromSource: + items: + properties: + path: + pattern: ^\/+.*[^\/]$ + type: string + type: object + type: array + message: + type: string + severity: + maximum: 10 + minimum: 1 + type: integer + tags: + items: + type: string + type: array + required: + - capability + - fromSource + type: object + type: array + message: + type: string + severity: + maximum: 10 + minimum: 1 + type: integer + tags: + items: + type: string + type: array + required: + - matchCapabilities + type: object + file: + properties: + action: + enum: + - Allow + - Audit + - Block + type: string + matchDirectories: + items: + properties: + action: + enum: + - Allow + - Audit + - Block + type: string + dir: + pattern: ^\/$|^\/.*\/$ + type: string + fromSource: + items: + properties: + path: + pattern: ^\/+.*[^\/]$ + type: string + type: object + type: array + message: + type: string + ownerOnly: + type: boolean + readOnly: + type: boolean + recursive: + type: boolean + severity: + maximum: 10 + minimum: 1 + type: integer + tags: + items: + type: string + type: array + required: + - dir + type: object + type: array + matchPaths: + items: + properties: + action: + enum: + - Allow + - Audit + - Block + type: string + fromSource: + items: + properties: + path: + pattern: ^\/+.*[^\/]$ + type: string + type: object + type: array + message: + type: string + ownerOnly: + type: boolean + path: + pattern: ^\/+.*[^\/]$ + type: string + readOnly: + type: boolean + severity: + maximum: 10 + minimum: 1 + type: integer + tags: + items: + type: string + type: array + required: + - path + type: object + type: array + matchPatterns: + items: + properties: + action: + enum: + - Allow + - Audit + - Block + type: string + message: + type: string + ownerOnly: + type: boolean + pattern: + type: string + readOnly: + type: boolean + severity: + maximum: 10 + minimum: 1 + type: integer + tags: + items: + type: string + type: array + required: + - pattern + type: object + type: array + message: + type: string + severity: + maximum: 10 + minimum: 1 + type: integer + tags: + items: + type: string + type: array + type: object + message: + type: string + network: + properties: + action: + enum: + - Allow + - Audit + - Block + type: string + matchProtocols: + items: + properties: + action: + enum: + - Allow + - Audit + - Block + type: string + fromSource: + items: + properties: + path: + pattern: ^\/+.*[^\/]$ + type: string + type: object + type: array + message: + type: string + protocol: + pattern: (icmp|ICMP|tcp|TCP|udp|UDP|raw|RAW)$ + type: string + severity: + maximum: 10 + minimum: 1 + type: integer + tags: + items: + type: string + type: array + required: + - fromSource + - protocol + type: object + type: array + message: + type: string + severity: + maximum: 10 + minimum: 1 + type: integer + tags: + items: + type: string + type: array + required: + - matchProtocols + type: object + nodeSelector: + properties: + matchLabels: + additionalProperties: + type: string + type: object + type: object + process: + properties: + action: + enum: + - Allow + - Audit + - Block + type: string + matchDirectories: + items: + properties: + action: + enum: + - Allow + - Audit + - Block + type: string + dir: + pattern: ^\/$|^\/.*\/$ + type: string + fromSource: + items: + properties: + path: + pattern: ^\/+.*[^\/]$ + type: string + type: object + type: array + message: + type: string + ownerOnly: + type: boolean + recursive: + type: boolean + severity: + maximum: 10 + minimum: 1 + type: integer + tags: + items: + type: string + type: array + required: + - dir + type: object + type: array + matchPaths: + items: + properties: + action: + enum: + - Allow + - Audit + - Block + type: string + fromSource: + items: + properties: + path: + pattern: ^\/+.*[^\/]$ + type: string + type: object + type: array + message: + type: string + ownerOnly: + type: boolean + path: + pattern: ^\/+.*[^\/]$ + type: string + severity: + maximum: 10 + minimum: 1 + type: integer + tags: + items: + type: string + type: array + required: + - path + type: object + type: array + matchPatterns: + items: + properties: + action: + enum: + - Allow + - Audit + - Block + type: string + message: + type: string + ownerOnly: + type: boolean + pattern: + type: string + severity: + maximum: 10 + minimum: 1 + type: integer + tags: + items: + type: string + type: array + required: + - pattern + type: object + type: array + message: + type: string + severity: + maximum: 10 + minimum: 1 + type: integer + tags: + items: + type: string + type: array + type: object + severity: + maximum: 10 + minimum: 1 + type: integer + tags: + items: + type: string + type: array + required: + - nodeSelector + type: object + status: + description: KubeArmorHostPolicyStatus defines the observed state of KubeArmorHostPolicy + properties: + status: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/KubeArmor/common/common.go b/KubeArmor/common/common.go index 2249159c01..58d5a04741 100644 --- a/KubeArmor/common/common.go +++ b/KubeArmor/common/common.go @@ -20,6 +20,22 @@ import ( kg "github.com/kubearmor/KubeArmor/KubeArmor/log" ) +// ContainerRuntimeSocketMap lists all sockets that are auto-detected for each +// supported runtime +var ContainerRuntimeSocketMap = map[string][]string{ + "docker": { + "/var/run/docker.sock", + }, + "containerd": { + "/var/snap/microk8s/common/run/containerd.sock", + "/run/k3s/containerd/containerd.sock", + "/var/run/containerd/containerd.sock", + }, + "crio": { + "/var/run/crio/crio.sock", + }, +} + // ============ // // == Common == // // ============ // @@ -365,6 +381,17 @@ func IsK8sEnv() bool { return false } +// GetCRISocket Function validates and returns the socket for each compatible +// runtime +func GetCRISocket(ContainerRuntime string) string { + for _, candidate := range ContainerRuntimeSocketMap[ContainerRuntime] { + if _, err := os.Stat(candidate); err == nil { + return candidate + } + } + return "" +} + // ==================== // // == Identity Match == // // ==================== // diff --git a/KubeArmor/config/config.go b/KubeArmor/config/config.go index 533d090390..5e5d212183 100644 --- a/KubeArmor/config/config.go +++ b/KubeArmor/config/config.go @@ -4,6 +4,7 @@ package config import ( + "errors" "fmt" "os" "strings" @@ -40,6 +41,8 @@ type KubearmorConfig struct { HostDefaultCapabilitiesPosture string // Default Enforcement Action in Global Capabilities Context CoverageTest bool // Enable/Disable Coverage Test + + CRISocket string // The container runtime endpoint to use } // PolicyDir policy dir path for host policies backup @@ -105,6 +108,9 @@ const ConfigCoverageTest string = "coverageTest" // ConfigK8sEnv VM key const ConfigK8sEnv string = "k8s" +// ConfigCRISocket key +const ConfigCRISocket string = "criSocket" + func readCmdLineParams() { hostname, _ := os.Hostname() clusterStr := flag.String(ConfigCluster, "default", "cluster name") @@ -132,6 +138,8 @@ func readCmdLineParams() { coverageTestB := flag.Bool(ConfigCoverageTest, false, "enabling CoverageTest") + criSocket := flag.String(ConfigCRISocket, "", "path to CRI socket. Format: unix:///path/to/file.sock. If empty kubearmor will try to auto-detect this.") + flags := []string{} flag.VisitAll(func(f *flag.Flag) { kv := fmt.Sprintf("%s:%v", f.Name, f.Value) @@ -165,6 +173,8 @@ func readCmdLineParams() { viper.SetDefault(ConfigHostDefaultCapabilitiesPosture, *hostDefaultCapabilitiesPosture) viper.SetDefault(ConfigCoverageTest, *coverageTestB) + + viper.SetDefault(ConfigCRISocket, *criSocket) } // LoadConfig Load configuration @@ -213,6 +223,16 @@ func LoadConfig() error { GlobalCfg.HostDefaultNetworkPosture = viper.GetString(ConfigHostDefaultNetworkPosture) GlobalCfg.HostDefaultCapabilitiesPosture = viper.GetString(ConfigHostDefaultCapabilitiesPosture) + // read CRI_SOCKET env variable. If empty, check criSocket flag + GlobalCfg.CRISocket = os.Getenv("CRI_SOCKET") + if GlobalCfg.CRISocket == "" { + GlobalCfg.CRISocket = viper.GetString(ConfigCRISocket) + } + + if GlobalCfg.CRISocket != "" && !strings.HasPrefix(GlobalCfg.CRISocket, "unix://") { + return errors.New(fmt.Sprintf("%s is invalid. CRI socket must start with unix://", GlobalCfg.CRISocket)) + } + kg.Printf("Configuration [%+v]", GlobalCfg) if GlobalCfg.KVMAgent { diff --git a/KubeArmor/core/containerdHandler.go b/KubeArmor/core/containerdHandler.go index dd9b7a894a..1e7defb231 100644 --- a/KubeArmor/core/containerdHandler.go +++ b/KubeArmor/core/containerdHandler.go @@ -7,8 +7,8 @@ import ( "context" "fmt" "os" - "path/filepath" "strconv" + "strings" "time" kl "github.com/kubearmor/KubeArmor/KubeArmor/common" @@ -42,8 +42,6 @@ func init() { typeurl.Register(&specs.Spec{}, prefix, "opencontainers/runtime-spec", major, "Spec") typeurl.Register(&specs.Process{}, prefix, "opencontainers/runtime-spec", major, "Process") - - Containerd = NewContainerdHandler() } // ContainerdHandler Structure @@ -72,29 +70,15 @@ type ContainerdHandler struct { func NewContainerdHandler() *ContainerdHandler { ch := &ContainerdHandler{} - sockFile := "unix://" - - for idx, candidate := range []string{"/var/run/containerd/containerd.sock", "/var/snap/microk8s/common/run/containerd.sock", "/run/k3s/containerd/containerd.sock"} { - if _, err := os.Stat(filepath.Clean(candidate)); err == nil { - sockFile = sockFile + candidate - - if idx == 0 { // containerd - ch.StoragePath = "/run/containerd" - } else if idx == 1 { // microk8s - ch.StoragePath = "/var/snap/microk8s/common/run/containerd" - } else if idx == 2 { // k3s - ch.StoragePath = "/run/k3s/containerd" - } - - break - } - } - - if sockFile == "unix://" { - return nil + if strings.Contains(cfg.GlobalCfg.CRISocket, "microk8s") { // microk8s + ch.StoragePath = "/var/snap/microk8s/common/run/containerd" + } else if strings.Contains(cfg.GlobalCfg.CRISocket, "k3s") { // k3s + ch.StoragePath = "/run/k3s/containerd" + } else { // vanilla containerd + ch.StoragePath = "/run/containerd" } - conn, err := grpc.Dial(sockFile, grpc.WithInsecure()) + conn, err := grpc.Dial(cfg.GlobalCfg.CRISocket, grpc.WithInsecure()) if err != nil { return nil } @@ -380,6 +364,8 @@ func (dm *KubeArmorDaemon) UpdateContainerdContainer(ctx context.Context, contai // MonitorContainerdEvents Function func (dm *KubeArmorDaemon) MonitorContainerdEvents() { + Containerd = NewContainerdHandler() + // check if Containerd exists if Containerd == nil { return diff --git a/KubeArmor/core/crioHandler.go b/KubeArmor/core/crioHandler.go new file mode 100644 index 0000000000..3eb891a5f7 --- /dev/null +++ b/KubeArmor/core/crioHandler.go @@ -0,0 +1,368 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2021 Authors of KubeArmor + +package core + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strconv" + "time" + + kl "github.com/kubearmor/KubeArmor/KubeArmor/common" + cfg "github.com/kubearmor/KubeArmor/KubeArmor/config" + kg "github.com/kubearmor/KubeArmor/KubeArmor/log" + tp "github.com/kubearmor/KubeArmor/KubeArmor/types" + spec "github.com/opencontainers/runtime-spec/specs-go" + "google.golang.org/grpc" + pb "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" +) + +// CrioHandler Structure +type CrioHandler struct { + // connection + conn *grpc.ClientConn + + // crio client + client pb.RuntimeServiceClient + + // containers is a map with empty value to have lookups in constant time + containers map[string]struct{} +} + +// CrioContainerInfo struct corresponds to CRI-O's container info returned +// with container status +type CrioContainerInfo struct { + SandboxID string `json:"sandboxID"` + Pid int `json:"pid"` + RuntimeSpec spec.Spec `json:"runtimeSpec"` + Privileged bool `json:"privileged"` +} + +// Crio Handler +var Crio *CrioHandler + +// NewCrioHandler Function creates a new Crio handler +func NewCrioHandler() *CrioHandler { + ch := &CrioHandler{} + + conn, err := grpc.Dial(cfg.GlobalCfg.CRISocket, grpc.WithInsecure()) + if err != nil { + return nil + } + + ch.conn = conn + + // The runtime service client can be used for all RPCs + ch.client = pb.NewRuntimeServiceClient(ch.conn) + + ch.containers = make(map[string]struct{}) + + return ch +} + +// Close the connection +func (ch *CrioHandler) Close() { + if ch.conn != nil { + if err := ch.conn.Close(); err != nil { + kg.Err(err.Error()) + } + } +} + +// ==================== // +// == Container Info == // +// ==================== // + +// GetContainerInfo Function gets info of a particular container +func (ch *CrioHandler) GetContainerInfo(ctx context.Context, containerID string) (tp.Container, error) { + // request to get status of specified container + // verbose has to be true to retrieve additional CRI specific info + req := &pb.ContainerStatusRequest{ + ContainerId: containerID, + Verbose: true, + } + + res, err := ch.client.ContainerStatus(ctx, req) + if err != nil { + return tp.Container{}, err + } + + container := tp.Container{} + + // == container base == // + resContainerStatus := res.Status + + container.ContainerID = resContainerStatus.Id + container.ContainerName = resContainerStatus.Metadata.Name + + container.NamespaceName = "Unknown" + container.EndPointName = "Unknown" + + // check container labels + containerLables := resContainerStatus.Labels + if val, ok := containerLables["io.kubernetes.pod.namespace"]; ok { + container.NamespaceName = val + } + if val, ok := containerLables["io.kubernetes.pod.name"]; ok { + container.EndPointName = val + } + + // extracting the runtime specific "info" + var containerInfo CrioContainerInfo + err = json.Unmarshal([]byte(res.Info["info"]), &containerInfo) + if err != nil { + return tp.Container{}, err + } + + // path to container's root storage + container.AppArmorProfile = containerInfo.RuntimeSpec.Process.ApparmorProfile + + // path to the rootfs + container.MergedDir = containerInfo.RuntimeSpec.Root.Path + + pid := strconv.Itoa(containerInfo.Pid) + + if data, err := os.Readlink("/proc/" + pid + "/ns/pid"); err == nil { + if _, err := fmt.Sscanf(data, "pid:[%d]\n", &container.PidNS); err != nil { + kg.Warnf("Unable to get PidNS (%s, %s, %s)", containerID, pid, err.Error()) + } + } else { + return container, err + } + + if data, err := os.Readlink("/proc/" + pid + "/ns/mnt"); err == nil { + if _, err := fmt.Sscanf(data, "mnt:[%d]\n", &container.MntNS); err != nil { + kg.Warnf("Unable to get MntNS (%s, %s, %s)", containerID, pid, err.Error()) + } + } else { + return container, err + } + + return container, nil +} + +// ================= // +// == CRIO Events == // +// ================= // + +// GetCrioContainers Function gets IDs of all containers +func (ch *CrioHandler) GetCrioContainers() (map[string]struct{}, error) { + containers := make(map[string]struct{}) + var err error + + req := pb.ListContainersRequest{} + + if containerList, err := ch.client.ListContainers(context.Background(), &req); err == nil { + for _, container := range containerList.Containers { + containers[container.Id] = struct{}{} + } + + return containers, nil + } + + return nil, err +} + +// GetNewCrioContainers Function gets new crio containers +func (ch *CrioHandler) GetNewCrioContainers(containers map[string]struct{}) map[string]struct{} { + newContainers := make(map[string]struct{}) + + for activeContainerID := range containers { + if _, ok := ch.containers[activeContainerID]; !ok { + newContainers[activeContainerID] = struct{}{} + } + } + + return newContainers +} + +// GetDeletedCrioContainers Function gets deleted crio containers +func (ch *CrioHandler) GetDeletedCrioContainers(containers map[string]struct{}) map[string]struct{} { + deletedContainers := make(map[string]struct{}) + + for globalContainerID := range ch.containers { + if _, ok := containers[globalContainerID]; !ok { + deletedContainers[globalContainerID] = struct{}{} + delete(ch.containers, globalContainerID) + } + } + + ch.containers = containers + + return deletedContainers +} + +// UpdateCrioContainer Function +func (dm *KubeArmorDaemon) UpdateCrioContainer(ctx context.Context, containerID, action string) bool { + if Crio == nil { + return false + } + + if action == "start" { + // get container info from client + container, err := Crio.GetContainerInfo(ctx, containerID) + if err != nil { + kg.Warnf(err.Error()) + return false + } + + if container.ContainerID == "" { + return false + } + + dm.ContainersLock.Lock() + if _, ok := dm.Containers[container.ContainerID]; !ok { + dm.Containers[container.ContainerID] = container + dm.ContainersLock.Unlock() + } else if dm.Containers[container.ContainerID].PidNS == 0 && dm.Containers[container.ContainerID].MntNS == 0 { + container.NamespaceName = dm.Containers[container.ContainerID].NamespaceName + container.EndPointName = dm.Containers[container.ContainerID].EndPointName + container.Labels = dm.Containers[container.ContainerID].Labels + + container.ContainerName = dm.Containers[container.ContainerID].ContainerName + container.ContainerImage = dm.Containers[container.ContainerID].ContainerImage + + container.PolicyEnabled = dm.Containers[container.ContainerID].PolicyEnabled + + container.ProcessVisibilityEnabled = dm.Containers[container.ContainerID].ProcessVisibilityEnabled + container.FileVisibilityEnabled = dm.Containers[container.ContainerID].FileVisibilityEnabled + container.NetworkVisibilityEnabled = dm.Containers[container.ContainerID].NetworkVisibilityEnabled + container.CapabilitiesVisibilityEnabled = dm.Containers[container.ContainerID].CapabilitiesVisibilityEnabled + + dm.Containers[container.ContainerID] = container + dm.ContainersLock.Unlock() + + dm.EndPointsLock.Lock() + for idx, endPoint := range dm.EndPoints { + if endPoint.NamespaceName == container.NamespaceName && endPoint.EndPointName == container.EndPointName { + // update containers + if !kl.ContainsElement(endPoint.Containers, container.ContainerID) { + dm.EndPoints[idx].Containers = append(dm.EndPoints[idx].Containers, container.ContainerID) + } + + // update apparmor profiles + if !kl.ContainsElement(endPoint.AppArmorProfiles, container.AppArmorProfile) { + dm.EndPoints[idx].AppArmorProfiles = append(dm.EndPoints[idx].AppArmorProfiles, container.AppArmorProfile) + } + + break + } + } + dm.EndPointsLock.Unlock() + } else { + dm.ContainersLock.Unlock() + return false + } + + if dm.SystemMonitor != nil && cfg.GlobalCfg.Policy { + // update NsMap + dm.SystemMonitor.AddContainerIDToNsMap(containerID, container.PidNS, container.MntNS) + } + + dm.Logger.Printf("Detected a container (added/%s)", containerID[:12]) + } else if action == "destroy" { + dm.ContainersLock.Lock() + container, ok := dm.Containers[containerID] + if !ok { + dm.ContainersLock.Unlock() + return false + } + delete(dm.Containers, containerID) + dm.ContainersLock.Unlock() + + dm.EndPointsLock.Lock() + for idx, endPoint := range dm.EndPoints { + if endPoint.NamespaceName == container.NamespaceName && endPoint.EndPointName == container.EndPointName { + // update containers + for idxC, containerID := range endPoint.Containers { + if containerID == container.ContainerID { + dm.EndPoints[idx].Containers = append(dm.EndPoints[idx].Containers[:idxC], dm.EndPoints[idx].Containers[idxC+1:]...) + break + } + } + + // update apparmor profiles + for idxA, profile := range endPoint.AppArmorProfiles { + if profile == container.AppArmorProfile { + dm.EndPoints[idx].AppArmorProfiles = append(dm.EndPoints[idx].AppArmorProfiles[:idxA], dm.EndPoints[idx].AppArmorProfiles[idxA+1:]...) + break + } + } + + break + } + } + dm.EndPointsLock.Unlock() + + if dm.SystemMonitor != nil && cfg.GlobalCfg.Policy { + // update NsMap + dm.SystemMonitor.DeleteContainerIDFromNsMap(containerID) + } + + dm.Logger.Printf("Detected a container (removed/%s)", containerID[:12]) + } + + return true +} + +// MonitorCrioEvents Function +func (dm *KubeArmorDaemon) MonitorCrioEvents() { + Crio = NewCrioHandler() + // check if Crio exists + if Crio == nil { + return + } + + dm.WgDaemon.Add(1) + defer dm.WgDaemon.Done() + + dm.Logger.Print("Started to monitor CRI-O events") + + for { + select { + case <-StopChan: + return + + default: + containers, err := Crio.GetCrioContainers() + if err != nil { + return + } + + // if number of stored container IDs is equal to number of container IDs + // returned by the API, no containers added/deleted + if len(containers) == len(Crio.containers) { + time.Sleep(time.Millisecond * 10) + continue + } + + invalidContainers := []string{} + + newContainers := Crio.GetNewCrioContainers(containers) + deletedContainers := Crio.GetDeletedCrioContainers(containers) + + if len(newContainers) > 0 { + for containerID := range newContainers { + if !dm.UpdateCrioContainer(context.Background(), containerID, "start") { + invalidContainers = append(invalidContainers, containerID) + } + } + } + + for _, invalidContainerID := range invalidContainers { + delete(Crio.containers, invalidContainerID) + } + + if len(deletedContainers) > 0 { + for containerID := range deletedContainers { + dm.UpdateCrioContainer(context.Background(), containerID, "destroy") + } + } + } + + time.Sleep(time.Millisecond * 50) + } +} diff --git a/KubeArmor/core/dockerHandler.go b/KubeArmor/core/dockerHandler.go index ef67d74ba1..761ca1e855 100644 --- a/KubeArmor/core/dockerHandler.go +++ b/KubeArmor/core/dockerHandler.go @@ -29,11 +29,6 @@ import ( // Docker Handler var Docker *DockerHandler -// init Function -func init() { - Docker = NewDockerHandler() -} - // DockerVersion Structure type DockerVersion struct { APIVersion string `json:"ApiVersion"` @@ -52,7 +47,7 @@ func NewDockerHandler() *DockerHandler { // specify the docker api version that we want to use // Versioned API: https://docs.docker.com/engine/api/ - versionStr, err := kl.GetCommandOutputWithErr("curl", []string{"--silent", "--unix-socket", "/var/run/docker.sock", "http://localhost/version"}) + versionStr, err := kl.GetCommandOutputWithErr("curl", []string{"--silent", "--unix-socket", strings.TrimPrefix(cfg.GlobalCfg.CRISocket, "unix://"), "http://localhost/version"}) if err != nil { return nil } @@ -381,6 +376,8 @@ func (dm *KubeArmorDaemon) MonitorDockerEvents() { dm.WgDaemon.Add(1) defer dm.WgDaemon.Done() + Docker = NewDockerHandler() + // check if Docker exists if Docker == nil { return diff --git a/KubeArmor/core/kubeArmor.go b/KubeArmor/core/kubeArmor.go index cb9906cbc7..34bd3299c3 100644 --- a/KubeArmor/core/kubeArmor.go +++ b/KubeArmor/core/kubeArmor.go @@ -11,6 +11,7 @@ import ( "syscall" "time" + "github.com/kubearmor/KubeArmor/KubeArmor/common" kl "github.com/kubearmor/KubeArmor/KubeArmor/common" cfg "github.com/kubearmor/KubeArmor/KubeArmor/config" kg "github.com/kubearmor/KubeArmor/KubeArmor/log" @@ -449,62 +450,103 @@ func KubeArmor() { // == // if dm.K8sEnabled && cfg.GlobalCfg.Policy { - if strings.HasPrefix(dm.Node.ContainerRuntimeVersion, "docker") { - sockFile := false + // check if the CRI socket set while executing kubearmor exists + if cfg.GlobalCfg.CRISocket != "" { + trimmedSocket := strings.TrimPrefix(cfg.GlobalCfg.CRISocket, "unix://") + if _, err := os.Stat(trimmedSocket); err != nil { + dm.Logger.Warnf("Error while looking for CRI socket file: %s", err.Error()) - for _, candidate := range []string{"/var/run/docker.sock"} { - if _, err := os.Stat(candidate); err == nil { - sockFile = true - break - } + // destroy the daemon + dm.DestroyKubeArmorDaemon() + return } - if sockFile { + // monitor containers + if strings.Contains(cfg.GlobalCfg.CRISocket, "docker") { // update already deployed containers dm.GetAlreadyDeployedDockerContainers() - // monitor docker events go dm.MonitorDockerEvents() + } else if strings.Contains(cfg.GlobalCfg.CRISocket, "crio") { + // monitor crio events + go dm.MonitorCrioEvents() + } else if strings.Contains(cfg.GlobalCfg.CRISocket, "containerd") { + // monitor containerd events + go dm.MonitorContainerdEvents() } else { - for _, candidate := range []string{"/var/run/containerd/containerd.sock"} { - if _, err := os.Stat(candidate); err == nil { - sockFile = true - break + dm.Logger.Errf("Failed to monitor containers: %s is not a supported CRI socket.", cfg.GlobalCfg.CRISocket) + // destroy the daemon + dm.DestroyKubeArmorDaemon() + return + } + + dm.Logger.Printf("Using %s for monitoring containers.", cfg.GlobalCfg.CRISocket) + + } else { // CRI socket not set, we'll have to auto detect + dm.Logger.Print("CRI socket not set. Trying to detect.") + + if strings.HasPrefix(dm.Node.ContainerRuntimeVersion, "docker") { + socketFile := common.GetCRISocket("docker") + + if socketFile != "" { + cfg.GlobalCfg.CRISocket = "unix://" + socketFile + + // update already deployed containers + dm.GetAlreadyDeployedDockerContainers() + + // monitor docker events + go dm.MonitorDockerEvents() + } else { + // we might have to use containerd's socket as docker's socket is not + // available + socketFile := common.GetCRISocket("containerd") + + if socketFile != "" { + cfg.GlobalCfg.CRISocket = "unix://" + socketFile + + // monitor containerd events + go dm.MonitorContainerdEvents() + } else { + dm.Logger.Err("Failed to monitor containers (Docker socket file is not accessible)") + + // destroy the daemon + dm.DestroyKubeArmorDaemon() + + return } } + } else if strings.HasPrefix(dm.Node.ContainerRuntimeVersion, "cri-o") { // cri-o + socketFile := common.GetCRISocket("crio") - if sockFile { - // monitor containerd events - go dm.MonitorContainerdEvents() + if socketFile != "" { + cfg.GlobalCfg.CRISocket = "unix://" + socketFile + + // monitor cri-o events + go dm.MonitorCrioEvents() } else { - dm.Logger.Err("Failed to monitor containers (Docker socket file is not accessible)") + dm.Logger.Err("Failed to monitor containers (CRI-O socket file is not accessible)") // destroy the daemon dm.DestroyKubeArmorDaemon() return } - } - } else { // containerd - sockFile := false + } else { // containerd + socketFile := common.GetCRISocket("containerd") - for _, candidate := range []string{"/var/run/containerd/containerd.sock", "/var/snap/microk8s/common/run/containerd.sock", "/run/k3s/containerd/containerd.sock"} { - if _, err := os.Stat(candidate); err == nil { - sockFile = true - break - } - } + if socketFile != "" { + cfg.GlobalCfg.CRISocket = "unix://" + socketFile - if sockFile { - // monitor containerd events - go dm.MonitorContainerdEvents() - } else { - dm.Logger.Err("Failed to monitor containers (Containerd socket file is not accessible)") + // monitor containerd events + go dm.MonitorContainerdEvents() + } else { + dm.Logger.Err("Failed to monitor containers (Containerd socket file is not accessible)") - // destroy the daemon - dm.DestroyKubeArmorDaemon() + // destroy the daemon + dm.DestroyKubeArmorDaemon() - return + return + } } } } diff --git a/KubeArmor/core/kubeUpdate.go b/KubeArmor/core/kubeUpdate.go index 847ee54b2f..ca0b342a2f 100644 --- a/KubeArmor/core/kubeUpdate.go +++ b/KubeArmor/core/kubeUpdate.go @@ -498,6 +498,10 @@ func (dm *KubeArmorDaemon) WatchK8sPods() { containerID := strings.TrimPrefix(container.ContainerID, "containerd://") pod.Containers[containerID] = container.Name pod.ContainerImages[containerID] = container.Image + kl.GetSHA256ofImage(container.ImageID) + } else if strings.HasPrefix(container.ContainerID, "cri-o://") { + containerID := strings.TrimPrefix(container.ContainerID, "cri-o://") + pod.Containers[containerID] = container.Name + pod.ContainerImages[containerID] = container.Image + kl.GetSHA256ofImage(container.ImageID) } } } diff --git a/KubeArmor/enforcer/appArmorEnforcer.go b/KubeArmor/enforcer/appArmorEnforcer.go index 430f31c72c..d68aa923e8 100644 --- a/KubeArmor/enforcer/appArmorEnforcer.go +++ b/KubeArmor/enforcer/appArmorEnforcer.go @@ -500,7 +500,7 @@ func (ae *AppArmorEnforcer) UpdateSecurityPolicies(endPoint tp.EndPoint) { appArmorProfiles := []string{} for _, appArmorProfile := range endPoint.AppArmorProfiles { - if kl.ContainsElement([]string{"docker-default", "unconfined", "cri-containerd.apparmor.d", ""}, appArmorProfile) { + if kl.ContainsElement([]string{"docker-default", "unconfined", "cri-containerd.apparmor.d", "crio-default", ""}, appArmorProfile) { continue } diff --git a/KubeArmor/go.mod b/KubeArmor/go.mod index 86aa2cc229..4abe87fedf 100644 --- a/KubeArmor/go.mod +++ b/KubeArmor/go.mod @@ -40,4 +40,5 @@ require ( k8s.io/api v0.21.2 k8s.io/apimachinery v0.21.2 k8s.io/client-go v0.21.2 + k8s.io/cri-api v0.24.0 ) diff --git a/KubeArmor/go.sum b/KubeArmor/go.sum index 6e0c42f183..a372b25c9f 100644 --- a/KubeArmor/go.sum +++ b/KubeArmor/go.sum @@ -106,6 +106,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= @@ -257,6 +258,7 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -749,8 +751,10 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -834,18 +838,22 @@ golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -943,8 +951,9 @@ google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a h1:pOwg4OoaRYScjmR4LlLgdtnyoHYTSAVhhqe5uPdpII8= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 h1:Et6SkiuvnBn+SgrSYXs/BrUpGB4mbdwt4R3vaPIlicA= +google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -961,6 +970,7 @@ google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8= google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -1047,6 +1057,8 @@ k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc= +k8s.io/cri-api v0.24.0 h1:PZ/MqhgYq4rxCarYe2rGNmd8G9ZuyS1NU9igolbkqlI= +k8s.io/cri-api v0.24.0/go.mod h1:t3tImFtGeStN+ES69bQUX9sFg67ek38BM9YIJhMmuig= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= diff --git a/contribution/k3s/install_k3s.sh b/contribution/k3s/install_k3s.sh index b02583a484..e870a6c878 100755 --- a/contribution/k3s/install_k3s.sh +++ b/contribution/k3s/install_k3s.sh @@ -3,7 +3,7 @@ # Copyright 2021 Authors of KubeArmor # create a single-node K3s cluster -if [ -x "$(command -v docker)" ]; then # docker +if [ "$RUNTIME" == "docker" ]; then # docker CGROUP_SYSTEMD=$(docker info 2> /dev/null | grep -i cgroup | grep systemd | wc -l) if [ $CGROUP_SYSTEMD == 1 ]; then curl -sfL https://get.k3s.io | K3S_KUBECONFIG_MODE="644" INSTALL_K3S_EXEC="--disable=traefik --docker --kubelet-arg cgroup-driver=systemd" sh - @@ -12,9 +12,12 @@ if [ -x "$(command -v docker)" ]; then # docker curl -sfL https://get.k3s.io | K3S_KUBECONFIG_MODE="644" INSTALL_K3S_EXEC="--disable=traefik --docker" sh - [[ $? != 0 ]] && echo "Failed to install k3s" && exit 1 fi -else # containerd - curl -sfL https://get.k3s.io | K3S_KUBECONFIG_MODE="644" INSTALL_K3S_EXEC="--disable=traefik" sh - - [[ $? != 0 ]] && echo "Failed to install k3s" && exit 1 +elif [ "$RUNTIME" == "crio" ]; then # cri-o + curl -sfL https://get.k3s.io | K3S_KUBECONFIG_MODE="644" INSTALL_K3S_EXEC="--disable=traefik --container-runtime-endpoint unix:///var/run/crio/crio.sock --kubelet-arg cgroup-driver=systemd" sh - + [[ $? != 0 ]] && echo "Failed to install k3s" && exit 1 +else # use containerd by default + curl -sfL https://get.k3s.io | K3S_KUBECONFIG_MODE="644" INSTALL_K3S_EXEC="--disable=traefik" sh - + [[ $? != 0 ]] && echo "Failed to install k3s" && exit 1 fi if [[ $(hostname) = kubearmor-dev* ]]; then diff --git a/contribution/self-managed-k8s-selinux/crio/install_crio.sh b/contribution/self-managed-k8s-selinux/crio/install_crio.sh new file mode 100755 index 0000000000..c9de69e515 --- /dev/null +++ b/contribution/self-managed-k8s-selinux/crio/install_crio.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2021 Authors of KubeArmor + +. /etc/os-release + +if [ "$ID" != "centos" ]; then + echo "Supports CentOS" + exit +fi + +OS="CentOS_${VERSION_ID}" +VERSION=1.19 + +if [ "$NAME" == "CentOS Stream" ]; then + OS="${OS}_Stream" +fi + +# remove podman +sudo yum remove buildah skopeo podman containers-common atomic-registries docker container-tools + +# remove left-over files +sudo rm -rf /etc/containers/* /var/lib/containers/* /etc/docker /etc/subuid* /etc/subgid* +cd ~ && rm -rf /.local/share/containers/ + +# disable selinux +sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config + +# setup repo +sudo curl -L -o /etc/yum.repos.d/devel:kubic:libcontainers:stable.repo https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/devel:kubic:libcontainers:stable.repo +sudo curl -L -o /etc/yum.repos.d/devel:kubic:libcontainers:stable:cri-o:$VERSION.repo https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable:cri-o:$VERSION/$OS/devel:kubic:libcontainers:stable:cri-o:$VERSION.repo + +sudo yum install cri-o containernetworking-plugins + +sudo systemctl daemon-reload +sudo systemctl start crio.service diff --git a/contribution/self-managed-k8s-selinux/crio/uninstall_crio.sh b/contribution/self-managed-k8s-selinux/crio/uninstall_crio.sh new file mode 100755 index 0000000000..3bae0a60d8 --- /dev/null +++ b/contribution/self-managed-k8s-selinux/crio/uninstall_crio.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2021 Authors of KubeArmor + +sudo systemctl stop crio.service + +sudo yum remove cri-o + +sudo rm -rf /etc/crictl.yaml +sudo rm -rf /var/lib/crio diff --git a/contribution/self-managed-k8s/crio/install-crio.sh b/contribution/self-managed-k8s/crio/install-crio.sh new file mode 100755 index 0000000000..3adf304bbd --- /dev/null +++ b/contribution/self-managed-k8s/crio/install-crio.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2021 Authors of KubeArmor + +. /etc/os-release + +if [ "$NAME" != "Ubuntu" ]; then + echo "Support Ubuntu 18.xx, 20.xx" + exit +fi + +OS="x${NAME}_${VERSION_ID}" +VERSION=1.19 + +# get signing keys +echo "deb [signed-by=/usr/share/keyrings/libcontainers-archive-keyring.gpg] https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list +echo "deb [signed-by=/usr/share/keyrings/libcontainers-crio-archive-keyring.gpg] https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$VERSION.list + +# add repositories +sudo mkdir -p /usr/share/keyrings +curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | sudo gpg --yes --dearmor -o /usr/share/keyrings/libcontainers-archive-keyring.gpg +curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/Release.key | sudo gpg --yes --dearmor -o /usr/share/keyrings/libcontainers-crio-archive-keyring.gpg + +# install +sudo apt-get update +sudo apt-get install cri-o cri-o-runc + +# this option is not supported in ubuntu 18.04 +if [ "$VERSION_ID" == "18.04" ]; then + sudo sed -i 's/,metacopy=on//g' /etc/containers/storage.conf +fi + +sudo systemctl daemon-reload +sudo systemctl start crio.service diff --git a/contribution/self-managed-k8s/crio/uninstall-crio.sh b/contribution/self-managed-k8s/crio/uninstall-crio.sh new file mode 100755 index 0000000000..05da0ac78d --- /dev/null +++ b/contribution/self-managed-k8s/crio/uninstall-crio.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2021 Authors of KubeArmor + +sudo systemctl stop crio.service + +sudo apt purge -y cri-o cri-o-runc +sudo apt autoremove -y --purge cri-o cri-o-runc + +sudo rm -rf /etc/crictl.yaml +sudo rm -rf /var/lib/crio + +# check storage.conf diff --git a/contribution/self-managed-k8s/k8s/initialize_kubernetes.sh b/contribution/self-managed-k8s/k8s/initialize_kubernetes.sh index 3eb8fda6d3..17375a50e5 100755 --- a/contribution/self-managed-k8s/k8s/initialize_kubernetes.sh +++ b/contribution/self-managed-k8s/k8s/initialize_kubernetes.sh @@ -7,9 +7,20 @@ if [ "$CNI" == "" ]; then CNI=cilium fi +# use docker as default CRI +if [ "$CRI_SOCKET" == "" ]; then + if [ -f /var/run/docker.sock ]; then + CRI_SOCKET=unix:///var/run/docker.sock + elif [ -f /var/run/containerd/containerd.sock ]; then + CRI_SOCKET=unix:///var/run/containerd/containerd.sock + elif [ -f /var/run/crio/crio.sock ]; then + CRI_SOCKET=unix:///var/run/crio/crio.sock + fi +fi + # check supported CNI if [ "$CNI" != "flannel" ] && [ "$CNI" != "weave" ] && [ "$CNI" != "calico" ] && [ "$CNI" != "cilium" ]; then - echo "Usage: CNI={flannel|weave|calico|cilium} MASTER={true|false} $0" + echo "Usage: CNI={flannel|weave|calico|cilium} CRI_SOCKET="unix:///path/to/socket_file" MASTER={true|false} $0" exit fi @@ -26,9 +37,9 @@ sudo bash -c "echo 'net.bridge.bridge-nf-call-iptables=1' >> /etc/sysctl.conf" # initialize the master node if [ "$CNI" == "calico" ]; then - sudo kubeadm init --pod-network-cidr=192.168.0.0/16 | tee -a ~/k8s_init.log + sudo kubeadm init --cri-socket=$CRI_SOCKET --pod-network-cidr=192.168.0.0/16 | tee -a ~/k8s_init.log else # weave, flannel, cilium - sudo kubeadm init --pod-network-cidr=10.244.0.0/16 | tee -a ~/k8s_init.log + sudo kubeadm init --cri-socket=$CRI_SOCKET --pod-network-cidr=10.244.0.0/16 | tee -a ~/k8s_init.log fi # make kubectl work for non-root user diff --git a/contribution/vagrant/Vagrantfile b/contribution/vagrant/Vagrantfile index 8446700687..ea40c17078 100644 --- a/contribution/vagrant/Vagrantfile +++ b/contribution/vagrant/Vagrantfile @@ -76,6 +76,13 @@ Vagrant.configure("2") do |config| # install Kubernetes config.vm.provision :shell, :inline => "RUNTIME=docker /home/vagrant/KubeArmor/contribution/self-managed-k8s-selinux/k8s/install_kubernetes.sh" + elsif ENV['RUNTIME'] == "crio" then + # install CRI-O + config.vm.provision :shell, path: kubearmor_home + "/contribution/self-managed-k8s-selinux/crio/install_crio.sh" + + # install Kubernetes + config.vm.provision :shell, :inline => "RUNTIME=crio /home/vagrant/KubeArmor/contribution/self-managed-k8s-selinux/k8s/install_kubernetes.sh" + else # default == 'docker' # install Docker config.vm.provision :shell, path: kubearmor_home + "/contribution/self-managed-k8s-selinux/docker/install_docker.sh" @@ -108,6 +115,13 @@ Vagrant.configure("2") do |config| # install Kubernetes config.vm.provision :shell, :inline => "RUNTIME=containerd /home/vagrant/KubeArmor/contribution/self-managed-k8s/k8s/install_kubernetes.sh" + elsif ENV['RUNTIME'] == "crio" then + # install CRI-O + config.vm.provision :shell, path: kubearmor_home + "/contribution/self-managed-k8s/crio/install-crio.sh" + + # install Kubernetes + config.vm.provision :shell, :inline => "CRI_SOCKET=unix:///var/run/crio/crio.sock /home/vagrant/KubeArmor/contribution/self-managed-k8s/k8s/install_kubernetes.sh" + else # default == 'docker' # install Docker config.vm.provision :shell, path: kubearmor_home + "/contribution/self-managed-k8s/docker/install_docker.sh"