From 4e65d923764d96e939b8ed5dbfbe99dc3b11c3af Mon Sep 17 00:00:00 2001 From: AbdelrahmanElawady Date: Sun, 21 Apr 2024 21:52:34 +0200 Subject: [PATCH] Detach a process to pass all containers to KubeArmor Signed-off-by: AbdelrahmanElawady --- KubeArmor/core/hook_handler.go | 86 ++++++- pkg/KubeArmorOperator/Dockerfile | 4 +- pkg/KubeArmorOperator/cmd/snitch-cmd/main.go | 36 ++- pkg/KubeArmorOperator/go.mod | 24 +- pkg/KubeArmorOperator/go.sum | 20 +- pkg/KubeArmorOperator/hook/crio.go | 106 +++++++++ pkg/KubeArmorOperator/hook/ka.json | 11 - pkg/KubeArmorOperator/hook/main.go | 232 ++++++++++++++++--- pkg/KubeArmorOperator/hook/types.go | 15 ++ 9 files changed, 466 insertions(+), 68 deletions(-) create mode 100644 pkg/KubeArmorOperator/hook/crio.go delete mode 100644 pkg/KubeArmorOperator/hook/ka.json create mode 100644 pkg/KubeArmorOperator/hook/types.go diff --git a/KubeArmor/core/hook_handler.go b/KubeArmor/core/hook_handler.go index 3117de9e1b..e62cea6c8e 100644 --- a/KubeArmor/core/hook_handler.go +++ b/KubeArmor/core/hook_handler.go @@ -10,6 +10,7 @@ import ( "net" "os" "path/filepath" + "sync/atomic" kl "github.com/kubearmor/KubeArmor/KubeArmor/common" cfg "github.com/kubearmor/KubeArmor/KubeArmor/config" @@ -18,6 +19,8 @@ import ( const kubearmorDir = "/var/run/kubearmor" +// ListenToHook starts listening on a UNIX socket and waits for container hooks +// to pass new containers func (dm *KubeArmorDaemon) ListenToHook() { if err := os.MkdirAll(kubearmorDir, 0750); err != nil { log.Fatal(err) @@ -32,6 +35,7 @@ func (dm *KubeArmorDaemon) ListenToHook() { defer socket.Close() defer os.Remove(listenPath) + ready := &atomic.Bool{} for { conn, err := socket.Accept() @@ -39,12 +43,17 @@ func (dm *KubeArmorDaemon) ListenToHook() { log.Fatal(err) } - go dm.handleConn(conn) + go dm.handleConn(conn, ready) } } -func (dm *KubeArmorDaemon) handleConn(conn net.Conn) { +// handleConn gets container details from container hooks. +func (dm *KubeArmorDaemon) handleConn(conn net.Conn, ready *atomic.Bool) { + // We need to makes sure that no containers accepted until all containers created before KubeArmor + // are sent first. This is done mainly to avoid race conditions between hooks sending in + // data that some containers were deleted only for process responsible for sending previous containers + // to send that these containers are created. Which will leave KubeArmor in an incorrect state. defer conn.Close() buf := make([]byte, 4096) @@ -59,6 +68,7 @@ func (dm *KubeArmorDaemon) handleConn(conn net.Conn) { data := struct { Operation string `json:"operation"` + Detached bool `json:"detached"` Container types.Container `json:"container"` }{} @@ -66,7 +76,30 @@ func (dm *KubeArmorDaemon) handleConn(conn net.Conn) { if err != nil { log.Fatal(err) } - dm.Logger.Printf("got container %s", data.Container.ContainerID) + + if data.Detached { + // we want KubeArmor to start accepting containers after + // all previous container are set + defer ready.Store(true) + } else if !ready.Load() { + _, err = conn.Write([]byte("err")) + if err == io.EOF { + return + } else if err != nil { + log.Println(err) + return + } + continue + } + + _, err = conn.Write([]byte("ok")) + if err == io.EOF { + return + } else if err != nil { + log.Println(err) + return + } + if data.Operation == "create" { dm.handleContainerCreate(data.Container) } else { @@ -75,18 +108,59 @@ func (dm *KubeArmorDaemon) handleConn(conn net.Conn) { } } func (dm *KubeArmorDaemon) handleContainerCreate(container types.Container) { + endpoint := types.EndPoint{} + + dm.Logger.Printf("added %s", container.ContainerID) dm.ContainersLock.Lock() + if _, ok := dm.Containers[container.ContainerID]; !ok { + dm.Containers[container.ContainerID] = container + dm.ContainersLock.Unlock() + } else if dm.Containers[container.ContainerID].PidNS == 0 && dm.Containers[container.ContainerID].MntNS == 0 { + c := dm.Containers[container.ContainerID] + c.MntNS = container.MntNS + c.PidNS = container.PidNS + c.AppArmorProfile = container.AppArmorProfile + dm.Containers[c.ContainerID] = c + dm.ContainersLock.Unlock() + + dm.EndPointsLock.Lock() + for idx, endPoint := range dm.EndPoints { + if endPoint.NamespaceName == container.NamespaceName && endPoint.EndPointName == container.EndPointName && kl.ContainsElement(endPoint.Containers, container.ContainerID) { + + // update apparmor profiles + if !kl.ContainsElement(endPoint.AppArmorProfiles, container.AppArmorProfile) { + dm.EndPoints[idx].AppArmorProfiles = append(dm.EndPoints[idx].AppArmorProfiles, container.AppArmorProfile) + } + + if container.Privileged && dm.EndPoints[idx].PrivilegedContainers != nil { + dm.EndPoints[idx].PrivilegedContainers[container.ContainerName] = struct{}{} + } + + endpoint = dm.EndPoints[idx] + + break + } + } + dm.EndPointsLock.Unlock() + } else { + dm.ContainersLock.Unlock() + } if len(dm.OwnerInfo) > 0 { container.Owner = dm.OwnerInfo[container.EndPointName] } - dm.Containers[container.ContainerID] = container - dm.Logger.Printf("added %s", container.ContainerID) - dm.ContainersLock.Unlock() if dm.SystemMonitor != nil && cfg.GlobalCfg.Policy { dm.SystemMonitor.AddContainerIDToNsMap(container.ContainerID, container.NamespaceName, container.PidNS, container.MntNS) dm.RuntimeEnforcer.RegisterContainer(container.ContainerID, container.PidNS, container.MntNS) + + if len(endpoint.SecurityPolicies) > 0 { // struct can be empty or no policies registered for the endpoint yet + dm.Logger.UpdateSecurityPolicies("ADDED", endpoint) + if dm.RuntimeEnforcer != nil && endpoint.PolicyEnabled == types.KubeArmorPolicyEnabled { + // enforce security policies + dm.RuntimeEnforcer.UpdateSecurityPolicies(endpoint) + } + } } } func (dm *KubeArmorDaemon) handleContainerStop(containerID string) { diff --git a/pkg/KubeArmorOperator/Dockerfile b/pkg/KubeArmorOperator/Dockerfile index 28b21aa645..d6035a752c 100644 --- a/pkg/KubeArmorOperator/Dockerfile +++ b/pkg/KubeArmorOperator/Dockerfile @@ -40,7 +40,7 @@ COPY $OPERATOR_DIR/hook hook # Build RUN CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} GO111MODULE=on go build -a -o operator cmd/operator/main.go RUN CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} GO111MODULE=on go build -a -o snitch cmd/snitch-cmd/main.go -RUN CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} GO111MODULE=on go build -a -o hook/hook hook/main.go +RUN CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} GO111MODULE=on go build -a -o hook/hook ./hook FROM redhat/ubi9-minimal as operator @@ -81,7 +81,7 @@ LABEL name="kubearmor-snitch" \ ARG OPERATOR_DIR=pkg/KubeArmorOperator COPY --from=builder /KubeArmor/$OPERATOR_DIR/snitch /snitch -COPY --from=builder /KubeArmor/$OPERATOR_DIR/hook /hook +COPY --from=builder /KubeArmor/$OPERATOR_DIR/hook/hook /hook COPY LICENSE /licenses/license.txt ENTRYPOINT ["/snitch"] diff --git a/pkg/KubeArmorOperator/cmd/snitch-cmd/main.go b/pkg/KubeArmorOperator/cmd/snitch-cmd/main.go index 3c36747325..a0a8cc84a1 100644 --- a/pkg/KubeArmorOperator/cmd/snitch-cmd/main.go +++ b/pkg/KubeArmorOperator/cmd/snitch-cmd/main.go @@ -14,7 +14,9 @@ import ( "strings" "github.com/kubearmor/KubeArmor/pkg/KubeArmorOperator/seccomp" + "github.com/opencontainers/runtime-spec/specs-go" + hooks "github.com/containers/common/pkg/hooks/1.0.0" "github.com/kubearmor/KubeArmor/pkg/KubeArmorOperator/common" "github.com/kubearmor/KubeArmor/pkg/KubeArmorOperator/enforcer" "github.com/kubearmor/KubeArmor/pkg/KubeArmorOperator/k8s" @@ -125,7 +127,7 @@ func snitch() { ociHooksLabel := "no" if runtime == "cri-o" { // only cri-o supported for now ociHooksLabel = "yes" - if err := applyCRIOHook(); err != nil { + if err := applyCRIOHook(socket); err != nil { Logger.Errorf("Failed to apply OCI hook: %s", err.Error()) ociHooksLabel = "no" } @@ -167,7 +169,7 @@ func snitch() { } } -func applyCRIOHook() error { +func applyCRIOHook(socket string) error { // TODO: hook path should be fetched from container runtime. This is the default path hookDir := "/usr/share/containers/oci/hooks.d/" if err := os.MkdirAll(hookDir, 0750); err != nil { @@ -177,13 +179,35 @@ func applyCRIOHook() error { if err != nil { return err } - src, err := os.Open("/hook/ka.json") + defer dst.Close() + always := true + hook := hooks.Hook{ + Version: "1.0.0", + Hook: specs.Hook{ + Path: "/usr/share/kubearmor/hook", + Args: []string{ + "/usr/share/kubearmor/hook", + "--runtime-socket", + socket, + "--k8s", + }, + }, + When: hooks.When{Always: &always}, + Stages: []string{ + "createRuntime", + "poststop", + }, + } + hookBytes, err := json.Marshal(hook) if err != nil { return err } - if _, err := io.Copy(dst, src); err != nil { + + _, err = dst.Write(hookBytes) + if err != nil { return err } + kaDir := "/usr/share/kubearmor" if err := os.MkdirAll(kaDir, 0750); err != nil { return err @@ -192,10 +216,12 @@ func applyCRIOHook() error { if err != nil { return err } - srcBin, err := os.Open("/hook/hook") + defer dstBin.Close() + srcBin, err := os.Open("/hook") if err != nil { return err } + defer srcBin.Close() if _, err := io.Copy(dstBin, srcBin); err != nil { return err } diff --git a/pkg/KubeArmorOperator/go.mod b/pkg/KubeArmorOperator/go.mod index 084e1dce0a..f8e180d336 100644 --- a/pkg/KubeArmorOperator/go.mod +++ b/pkg/KubeArmorOperator/go.mod @@ -14,20 +14,32 @@ require ( github.com/kubearmor/KubeArmor/KubeArmor v0.0.0-20240110164432-c2c1b121cd94 github.com/kubearmor/KubeArmor/deployments v0.0.0-20230809083125-e2d5d5709d2c github.com/kubearmor/KubeArmor/pkg/KubeArmorController v0.0.0-20240110164432-c2c1b121cd94 - github.com/opencontainers/runtime-spec v1.1.0 + github.com/opencontainers/runtime-spec v1.2.0 github.com/spf13/cobra v1.8.0 go.uber.org/zap v1.26.0 + google.golang.org/grpc v1.63.2 k8s.io/api v0.29.0 k8s.io/apiextensions-apiserver v0.29.0 k8s.io/apimachinery v0.29.0 k8s.io/client-go v0.29.0 + k8s.io/cri-api v0.29.0 k8s.io/klog/v2 v2.120.0 k8s.io/kubectl v0.27.4 sigs.k8s.io/controller-runtime v0.15.3 ) +require ( + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect + golang.org/x/net v0.24.0 // indirect + golang.org/x/oauth2 v0.17.0 // indirect + golang.org/x/sys v0.19.0 // indirect + golang.org/x/term v0.19.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect +) + require ( github.com/cilium/ebpf v0.12.3 // indirect + github.com/containers/common v0.58.1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.11.2 // indirect github.com/evanphx/json-patch v5.7.0+incompatible // indirect @@ -35,13 +47,12 @@ require ( github.com/go-logr/logr v1.4.1 // indirect github.com/go-openapi/jsonpointer v0.20.2 // indirect github.com/go-openapi/jsonreference v0.20.4 // indirect - github.com/go-openapi/swag v0.22.9 // indirect + github.com/go-openapi/swag v0.22.10 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20230323073829-e72429f035bd // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.16 // indirect @@ -55,7 +66,6 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/onsi/ginkgo/v2 v2.14.0 // indirect github.com/pelletier/go-toml/v2 v2.1.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect @@ -65,14 +75,8 @@ require ( github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.18.2 // indirect - github.com/stretchr/testify v1.9.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20240119083558-1b970713d09a // indirect - golang.org/x/net v0.24.0 // indirect - golang.org/x/oauth2 v0.17.0 // indirect - golang.org/x/sys v0.19.0 // indirect - golang.org/x/term v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/appengine v1.6.8 // indirect diff --git a/pkg/KubeArmorOperator/go.sum b/pkg/KubeArmorOperator/go.sum index c1d094516d..40238d300c 100644 --- a/pkg/KubeArmorOperator/go.sum +++ b/pkg/KubeArmorOperator/go.sum @@ -1,5 +1,7 @@ github.com/cilium/ebpf v0.12.3 h1:8ht6F9MquybnY97at+VDZb3eQQr8ev79RueWeVaEcG4= github.com/cilium/ebpf v0.12.3/go.mod h1:TctK1ivibvI3znr66ljgi4hqOT8EYQjz1KWBfb1UVgM= +github.com/containers/common v0.58.1 h1:E1DN9Lr7kgMVQy7AXLv1CYQCiqnweklMiYWbf0KOnqY= +github.com/containers/common v0.58.1/go.mod h1:l3vMqanJGj7tZ3W/i76gEJ128VXgFUO1tLaohJXPvdk= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -19,8 +21,8 @@ github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbX github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdXSSgNeAhojU= github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4= -github.com/go-openapi/swag v0.22.9 h1:XX2DssF+mQKM2DHsbgZK74y/zj4mo9I99+89xUmuZCE= -github.com/go-openapi/swag v0.22.9/go.mod h1:3/OXnFfnMAwBD099SwYRk7GD3xOrr1iL7d/XNLXVVwE= +github.com/go-openapi/swag v0.22.10 h1:4y86NVn7Z2yYd6pfS4Z+Nyh3aAUL3Nul+LMbhFKy0gA= +github.com/go-openapi/swag v0.22.10/go.mod h1:Cnn8BYtRlx6BNE3DPN86f/xkapGIcLWzh3CLEb4C1jI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -77,8 +79,8 @@ github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY github.com/onsi/ginkgo/v2 v2.14.0/go.mod h1:JkUdW7JkN0V6rFvsHcJ478egV3XH9NxpD27Hal/PhZw= github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= -github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg= -github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= +github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -129,8 +131,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA= -golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -182,6 +184,10 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= +google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= @@ -207,6 +213,8 @@ k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o= k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMeis= k8s.io/client-go v0.29.0 h1:KmlDtFcrdUzOYrBhXHgKw5ycWzc3ryPX5mQe0SkG3y8= k8s.io/client-go v0.29.0/go.mod h1:yLkXH4HKMAywcrD82KMSmfYg2DlE8mepPR4JGSo5n38= +k8s.io/cri-api v0.29.0 h1:atenAqOltRsFqcCQlFFpDnl/R4aGfOELoNLTDJfd7t8= +k8s.io/cri-api v0.29.0/go.mod h1:Rls2JoVwfC7kW3tndm7267kriuRukQ02qfht0PCRuIc= k8s.io/klog/v2 v2.120.0 h1:z+q5mfovBj1fKFxiRzsa2DsJLPIVMk/KFL81LMOfK+8= k8s.io/klog/v2 v2.120.0/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240105020646-a37d4de58910 h1:1Rp/XEKP5uxPs6QrsngEHAxBjaAR78iJRiJq5Fi7LSU= diff --git a/pkg/KubeArmorOperator/hook/crio.go b/pkg/KubeArmorOperator/hook/crio.go new file mode 100644 index 0000000000..c581e56787 --- /dev/null +++ b/pkg/KubeArmorOperator/hook/crio.go @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2022 Authors of KubeArmor + +package main + +import ( + "context" + "encoding/json" + + "github.com/kubearmor/KubeArmor/KubeArmor/types" + "github.com/opencontainers/runtime-spec/specs-go" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" +) + +type crioHandler struct { + client runtime.RuntimeServiceClient + conn *grpc.ClientConn +} + +func newCRIOHandler(socket string) (handler, error) { + conn, err := grpc.Dial( + socket, + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + return nil, err + } + + client := runtime.NewRuntimeServiceClient(conn) + return &crioHandler{client: client, conn: conn}, nil +} + +func (h *crioHandler) close() { + _ = h.conn.Close() +} + +func (h *crioHandler) listContainers(ctx context.Context) ([]types.Container, error) { + containers := make([]types.Container, 0) + containersList, err := h.client.ListContainers(ctx, &runtime.ListContainersRequest{}) + if err != nil { + return nil, err + } + for _, container := range containersList.GetContainers() { + c, err := h.getContainer(ctx, container.GetId()) + if err != nil { + return nil, err + } + containers = append(containers, c) + } + return containers, nil +} + +func (h *crioHandler) getContainer(ctx context.Context, id string) (types.Container, error) { + containerStatus, err := h.client.ContainerStatus( + ctx, + &runtime.ContainerStatusRequest{ContainerId: id, Verbose: true}, + ) + if err != nil { + return types.Container{}, err + } + c, err := containerFromContainerStatus(containerStatus.Status, containerStatus.Info["info"]) + if err != nil { + return types.Container{}, err + } + return c, nil +} + +func containerFromContainerStatus(status *runtime.ContainerStatus, info string) (types.Container, error) { + container := types.Container{} + + container.ContainerID = status.Id + container.ContainerName = status.Metadata.Name + + container.NamespaceName = "Unknown" + container.EndPointName = "Unknown" + + containerLables := status.Labels + if val, ok := containerLables["io.kubernetes.pod.namespace"]; ok { + container.NamespaceName = val + } + if val, ok := containerLables["io.kubernetes.pod.name"]; ok { + container.EndPointName = val + } + + var containerInfo containerInfo + err := json.Unmarshal([]byte(info), &containerInfo) + if err != nil { + return container, err + } + + container.AppArmorProfile = containerInfo.RuntimeSpec.Process.ApparmorProfile + container.Privileged = containerInfo.Privileged + + container.PidNS, container.MntNS = getNS(containerInfo.Pid) + + return container, nil +} + +type containerInfo struct { + SandboxID string `json:"sandboxID"` + Pid int `json:"pid"` + RuntimeSpec specs.Spec `json:"runtimeSpec"` + Privileged bool `json:"privileged"` +} diff --git a/pkg/KubeArmorOperator/hook/ka.json b/pkg/KubeArmorOperator/hook/ka.json deleted file mode 100644 index cef12f47ad..0000000000 --- a/pkg/KubeArmorOperator/hook/ka.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "version": "1.0.0", - "hook": { - "path": "/usr/share/kubearmor/hook", - "args": ["hook"] - }, - "when": { - "always": true - }, - "stages": ["createRuntime", "poststop"] - } \ No newline at end of file diff --git a/pkg/KubeArmorOperator/hook/main.go b/pkg/KubeArmorOperator/hook/main.go index 738c9e53dd..3b6c074986 100644 --- a/pkg/KubeArmorOperator/hook/main.go +++ b/pkg/KubeArmorOperator/hook/main.go @@ -4,14 +4,19 @@ package main import ( + "bytes" + "context" "encoding/json" "flag" "fmt" "io" + "log" "net" "os" + "os/exec" "path/filepath" "strings" + "time" "github.com/kubearmor/KubeArmor/KubeArmor/types" "github.com/opencontainers/runtime-spec/specs-go" @@ -19,71 +24,187 @@ import ( var ( kubeArmorSocket string + runtimeSocket string + k8s bool + detached bool ) func main() { flag.StringVar(&kubeArmorSocket, "kubearmor-socket", "/var/run/kubearmor/ka.sock", "KubeArmor socket") + flag.StringVar(&runtimeSocket, "runtime-socket", "", "container runtime socket") + flag.BoolVar(&k8s, "k8s", false, "kubernetes environment") + flag.BoolVar(&detached, "detached", false, "run detached") flag.Parse() + + if runtimeSocket == "" { + log.Println("runtime socket must be set") + os.Exit(1) + } + if !strings.HasPrefix(runtimeSocket, "unix://") { + runtimeSocket = "unix://" + runtimeSocket + } + if detached { + if err := runDetached(); err != nil { + log.Println(err) + os.Exit(1) + } + os.Exit(0) + } input, err := io.ReadAll(os.Stdin) if err != nil { - fmt.Fprintln(os.Stderr, err) + log.Println(err) os.Exit(1) } state := specs.State{} err = json.Unmarshal(input, &state) if err != nil { - fmt.Fprintln(os.Stderr, err) + log.Println(err) os.Exit(1) } + if err := run(state); err != nil { - fmt.Fprintln(os.Stderr, err) + log.Println(err) os.Exit(1) } } -func run(state specs.State) error { - container := types.Container{ - ContainerID: state.ID, - ContainerName: state.Annotations["io.kubernetes.container.name"], - NamespaceName: state.Annotations["io.kubernetes.pod.namespace"], - EndPointName: state.Annotations["io.kubernetes.pod.name"], +func runDetached() error { + // we need to make sure the process exits at some point + time.AfterFunc(1*time.Minute, func() { + log.Println("failed to get containers, process timed out") + os.Exit(1) + }) + conn := waitOnKubeArmor() + defer conn.Close() + + handler, err := newCRIOHandler(runtimeSocket) + if err != nil { + return err + } + containers, err := handler.listContainers(context.Background()) + if err != nil { + return err } - container.AppArmorProfile = state.Annotations[fmt.Sprintf("container.apparmor.security.beta.kubernetes.io/%s", container.ContainerName)] - container.AppArmorProfile = strings.TrimPrefix(container.AppArmorProfile, "localhost/") - if state.Status != specs.StateStopped { - nsPath := fmt.Sprintf("/proc/%d/ns", state.Pid) + for _, container := range containers { + data := struct { + Operation string `json:"operation"` + Detached bool `json:"detached"` + Container types.Container `json:"container"` + }{Operation: "create", Detached: true, Container: container} - pidLink, err := os.Readlink(filepath.Join(nsPath, "pid")) + dataJSON, err := json.Marshal(data) if err != nil { return err } - if _, err := fmt.Sscanf(pidLink, "pid:[%d]\n", &container.PidNS); err != nil { + + _, err = conn.Write(dataJSON) + if err != nil { return err } - mntLink, err := os.Readlink(filepath.Join(nsPath, "mnt")) + ack := make([]byte, 1024) + _, err = conn.Read(ack) + if err == io.EOF { + return nil + } if err != nil { return err } - if _, err := fmt.Sscanf(mntLink, "mnt:[%d]\n", &container.MntNS); err != nil { + } + + return nil +} + +func run(state specs.State) error { + var container types.Container + operation := "create" + // we try to connect to runtime here to make sure the socket is correct + // before spawning a detached process + handler, err := newCRIOHandler(runtimeSocket) + if err != nil { + return err + } + handler.close() + + container.ContainerID = state.ID + if state.Status == specs.StateStopped { + operation = "delete" + return sendContainer(container, operation) + } + + var appArmorProfile string + // the decision whether a container is KubeArmor container or not is done + // based on two things: + // - if we managed to get container spec, then we check the init process + // - if we couldn't, we use container name from kubernetes annotations + // this might lead to some containers acting as KubeArmor to spawn a detached + // processes that are unneeded. However, the design of KubeArmor hook logic + // is built around being idempotent so same requests being sent over and over + // shouldn't be a security issue. We can always add more restrictions on that guess + // but we always need to make sure to never introduce any false negatives as KubeArmor + // running without knowledge of previous containers could be a security issue. + var isKubeArmor bool + specBytes, err := os.ReadFile(filepath.Join(state.Bundle, "config.json")) + if err != nil { + // revert back to annotations + containerName := state.Annotations["io.kubernetes.container.name"] + appArmorProfile = strings.TrimPrefix( + state.Annotations[fmt.Sprintf("container.apparmor.security.beta.kubernetes.io/%s", containerName)], + "localhost/", + ) + isKubeArmor = containerName == "kubearmor" + } else { + var spec specs.Spec + err = json.Unmarshal(specBytes, &spec) + if err != nil { return err } + appArmorProfile = spec.Process.ApparmorProfile // check if Process is nil?? + isKubeArmor = spec.Process.Args[0] == "/KubeArmor/kubearmor" } - data := struct { - Operation string `json:"operation"` - Container types.Container `json:"container"` - }{Operation: "create", Container: container} - if state.Status == specs.StateStopped { - data.Operation = "stop" + if isKubeArmor { + err = startDetachedProcess() + if err != nil { + return err + } + // we still continue to try to send container details after starting the detached process + // to make sure if it was a false positive (container trying to act as KubeArmor), we still + // monitor it. + } + container = types.Container{ + ContainerID: state.ID, + AppArmorProfile: appArmorProfile, + } + container.PidNS, container.MntNS = getNS(state.Pid) + + return sendContainer(container, operation) +} + +func getNS(pid int) (uint32, uint32) { + var pidNS uint32 + var mntNS uint32 + + nsPath := fmt.Sprintf("/proc/%d/ns", pid) + pidLink, err := os.Readlink(filepath.Join(nsPath, "pid")) + if err == nil { + if _, err := fmt.Sscanf(pidLink, "pid:[%d]\n", &pidNS); err != nil { + log.Println(err) + } } - dataJSON, err := json.Marshal(data) - if err != nil { - return err + + mntLink, err := os.Readlink(filepath.Join(nsPath, "mnt")) + if err == nil { + if _, err := fmt.Sscanf(mntLink, "mnt:[%d]\n", &mntNS); err != nil { + log.Println(err) + } } + return pidNS, mntNS +} +func sendContainer(container types.Container, operation string) error { conn, err := net.Dial("unix", kubeArmorSocket) if err != nil { // not returning error here because this can happen in multiple cases @@ -92,11 +213,66 @@ func run(state specs.State) error { // - KubeArmor crashed so there is nothing listening on socket return nil } + defer conn.Close() - _, err = conn.Write(dataJSON) + data := struct { + Operation string `json:"operation"` + Detached bool `json:"detached"` + Container types.Container `json:"container"` + }{Operation: operation, Detached: false, Container: container} + + dataJSON, err := json.Marshal(data) if err != nil { return err } - return nil + + for { + _, err = conn.Write(dataJSON) + if err != nil { + return err + } + ack := make([]byte, 1024) + n, err := conn.Read(ack) + if err == io.EOF { + return nil + } else if err != nil { + return err + } + response := ack[:n] + if bytes.Equal(response, []byte("ok")) { + return nil + } else { + time.Sleep(50 * time.Millisecond) // try again in 50 ms + continue + } + + } +} + +func waitOnKubeArmor() net.Conn { + for { + conn, err := net.Dial("unix", kubeArmorSocket) + if err == nil { + return conn + } + time.Sleep(500 * time.Millisecond) + } +} + +func startDetachedProcess() error { + args := os.Args[1:] + args = append(args, "--detached") + cmd := exec.Command(os.Args[0], args...) + logFile, err := os.OpenFile("/var/log/ka-hook.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return err + } + cmd.Stdout = logFile + cmd.Stderr = logFile + err = cmd.Start() + if err != nil { + return err + } + return cmd.Process.Release() } diff --git a/pkg/KubeArmorOperator/hook/types.go b/pkg/KubeArmorOperator/hook/types.go new file mode 100644 index 0000000000..aefad8fd36 --- /dev/null +++ b/pkg/KubeArmorOperator/hook/types.go @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2022 Authors of KubeArmor + +package main + +import ( + "context" + + "github.com/kubearmor/KubeArmor/KubeArmor/types" +) + +type handler interface { + listContainers(ctx context.Context) ([]types.Container, error) + close() +}