Skip to content

Commit

Permalink
Listen to hook messages and update snitch to setup hooks on host
Browse files Browse the repository at this point in the history
Signed-off-by: AbdelrahmanElawady <[email protected]>
  • Loading branch information
AbdelrahmanElawady committed Apr 24, 2024
1 parent b7baaf8 commit 31bf733
Show file tree
Hide file tree
Showing 12 changed files with 369 additions and 18 deletions.
9 changes: 8 additions & 1 deletion KubeArmor/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@ type KubearmorConfig struct {
DefaultPostureLogs bool // Enable/Disable Default Posture logs for AppArmor LSM
InitTimeout string // Timeout for main thread init stages

StateAgent bool // enable KubeArmor state agent
StateAgent bool // enable KubeArmor state agent
UseOCIHooks bool
}

// GlobalCfg Global configuration for Kubearmor
Expand Down Expand Up @@ -96,6 +97,7 @@ const (
ConfigDefaultPostureLogs string = "defaultPostureLogs"
ConfigInitTimeout string = "initTimeout"
ConfigStateAgent string = "enableKubeArmorStateAgent"
UseOCIHooks string = "useOCIHooks"
)

func readCmdLineParams() {
Expand Down Expand Up @@ -143,6 +145,7 @@ func readCmdLineParams() {
initTimeout := flag.String(ConfigInitTimeout, "60s", "Timeout for main thread init stages")

stateAgent := flag.Bool(ConfigStateAgent, false, "enabling KubeArmor State Agent client")
useOCIHooks := flag.Bool(UseOCIHooks, false, "Use OCI hooks to get new containers instead of using container runtime socket")

flags := []string{}
flag.VisitAll(func(f *flag.Flag) {
Expand Down Expand Up @@ -197,6 +200,8 @@ func readCmdLineParams() {
viper.SetDefault(ConfigInitTimeout, *initTimeout)

viper.SetDefault(ConfigStateAgent, *stateAgent)

viper.SetDefault(UseOCIHooks, *useOCIHooks)
}

// LoadConfig Load configuration
Expand Down Expand Up @@ -292,6 +297,8 @@ func LoadConfig() error {

GlobalCfg.StateAgent = viper.GetBool(ConfigStateAgent)

GlobalCfg.UseOCIHooks = viper.GetBool(UseOCIHooks)

kg.Printf("Final Configuration [%+v]", GlobalCfg)

return nil
Expand Down
126 changes: 126 additions & 0 deletions KubeArmor/core/hook_handler.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
// SPDX-License-Identifier: Apache-2.0
// Copyright 2022 Authors of KubeArmor

package core

import (
"encoding/json"
"io"
"log"
"net"
"os"
"path/filepath"

kl "github.com/kubearmor/KubeArmor/KubeArmor/common"
cfg "github.com/kubearmor/KubeArmor/KubeArmor/config"
"github.com/kubearmor/KubeArmor/KubeArmor/types"
)

const kubearmorDir = "/var/run/kubearmor"

func (dm *KubeArmorDaemon) ListenToHook() {
if err := os.MkdirAll(kubearmorDir, 0750); err != nil {
log.Fatal(err)
}

listenPath := filepath.Join(kubearmorDir, "ka.sock")
_ = os.Remove(listenPath) // in case kubearmor crashed and the socket wasn't removed
socket, err := net.Listen("unix", listenPath)
if err != nil {
log.Fatal(err)
}

defer socket.Close()
defer os.Remove(listenPath)

for {
conn, err := socket.Accept()
if err != nil {
log.Fatal(err)
}

go dm.handleConn(conn)
}

}

func (dm *KubeArmorDaemon) handleConn(conn net.Conn) {
defer conn.Close()
buf := make([]byte, 4096)

for {
n, err := conn.Read(buf)
if err == io.EOF {
return
}
if err != nil {
log.Fatal(err)
}

data := struct {
Operation string `json:"operation"`
Container types.Container `json:"container"`
}{}

err = json.Unmarshal(buf[:n], &data)
if err != nil {
log.Fatal(err)
}
dm.Logger.Printf("got container %s", data.Container.ContainerID)
if data.Operation == "create" {
dm.handleContainerCreate(data.Container)
} else {
dm.handleContainerStop(data.Container.ContainerID)
}
}
}
func (dm *KubeArmorDaemon) handleContainerCreate(container types.Container) {

dm.ContainersLock.Lock()
if len(dm.OwnerInfo) > 0 {
container.Owner = dm.OwnerInfo[container.EndPointName]
}
dm.Containers[container.ContainerID] = container
dm.Logger.Printf("added %s", container.ContainerID)
dm.ContainersLock.Unlock()

if dm.SystemMonitor != nil && cfg.GlobalCfg.Policy {
dm.SystemMonitor.AddContainerIDToNsMap(container.ContainerID, container.NamespaceName, container.PidNS, container.MntNS)
dm.RuntimeEnforcer.RegisterContainer(container.ContainerID, container.PidNS, container.MntNS)
}
}
func (dm *KubeArmorDaemon) handleContainerStop(containerID string) {
dm.ContainersLock.Lock()
container, ok := dm.Containers[containerID]
dm.Logger.Printf("deleted %s", containerID)
if !ok {
dm.ContainersLock.Unlock()
return
}
delete(dm.Containers, containerID)
dm.ContainersLock.Unlock()

dm.EndPointsLock.Lock()
for idx, endPoint := range dm.EndPoints {
if endPoint.NamespaceName == container.NamespaceName && endPoint.EndPointName == container.EndPointName && kl.ContainsElement(endPoint.Containers, container.ContainerID) {

// update apparmor profiles
for idxA, profile := range endPoint.AppArmorProfiles {
if profile == container.AppArmorProfile {
dm.EndPoints[idx].AppArmorProfiles = append(dm.EndPoints[idx].AppArmorProfiles[:idxA], dm.EndPoints[idx].AppArmorProfiles[idxA+1:]...)
break
}
}

break
}
}
dm.EndPointsLock.Unlock()

if dm.SystemMonitor != nil && cfg.GlobalCfg.Policy {
// update NsMap
dm.SystemMonitor.DeleteContainerIDFromNsMap(containerID, container.NamespaceName, container.PidNS, container.MntNS)
dm.RuntimeEnforcer.UnregisterContainer(containerID)
}

}
6 changes: 4 additions & 2 deletions KubeArmor/core/kubeArmor.go
Original file line number Diff line number Diff line change
Expand Up @@ -597,8 +597,10 @@ func KubeArmor() {
}

if dm.K8sEnabled && cfg.GlobalCfg.Policy {
// check if the CRI socket set while executing kubearmor exists
if cfg.GlobalCfg.CRISocket != "" {
if cfg.GlobalCfg.UseOCIHooks {
go dm.ListenToHook()

} else if cfg.GlobalCfg.CRISocket != "" { // check if the CRI socket set while executing kubearmor exists
trimmedSocket := strings.TrimPrefix(cfg.GlobalCfg.CRISocket, "unix://")
if _, err := os.Stat(trimmedSocket); err != nil {
dm.Logger.Warnf("Error while looking for CRI socket file: %s", err.Error())
Expand Down
3 changes: 3 additions & 0 deletions pkg/KubeArmorOperator/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,12 @@ COPY $OPERATOR_DIR/enforcer enforcer
COPY $OPERATOR_DIR/k8s k8s
COPY $OPERATOR_DIR/runtime runtime
COPY $OPERATOR_DIR/seccomp seccomp
COPY $OPERATOR_DIR/hook hook

# Build
RUN CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} GO111MODULE=on go build -a -o operator cmd/operator/main.go
RUN CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} GO111MODULE=on go build -a -o snitch cmd/snitch-cmd/main.go
RUN CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} GO111MODULE=on go build -a -o hook/hook hook/main.go

FROM redhat/ubi9-minimal as operator

Expand Down Expand Up @@ -79,6 +81,7 @@ LABEL name="kubearmor-snitch" \

ARG OPERATOR_DIR=pkg/KubeArmorOperator
COPY --from=builder /KubeArmor/$OPERATOR_DIR/snitch /snitch
COPY --from=builder /KubeArmor/$OPERATOR_DIR/hook /hook
COPY LICENSE /licenses/license.txt

ENTRYPOINT ["/snitch"]
45 changes: 45 additions & 0 deletions pkg/KubeArmorOperator/cmd/snitch-cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"context"
"encoding/json"
"errors"
"io"
"os"
"path/filepath"
"strings"
Expand Down Expand Up @@ -121,6 +122,14 @@ func snitch() {
Logger.Errorf("Not able to detect runtime")
os.Exit(1)
}
ociHooksLabel := "no"
if runtime == "cri-o" { // only cri-o supported for now
ociHooksLabel = "yes"
if err := applyCRIOHook(); err != nil {
Logger.Errorf("Failed to apply OCI hook: %s", err.Error())
ociHooksLabel = "no"
}
}

// Check BTF support
btfPresent := enforcer.CheckBtfSupport(PathPrefix, *Logger)
Expand All @@ -135,6 +144,7 @@ func snitch() {
patchNode.Metadata.Labels[common.RandLabel] = rand.String(4)
patchNode.Metadata.Labels[common.BTFLabel] = btfPresent
patchNode.Metadata.Labels[common.ApparmorFsLabel] = enforcer.CheckIfApparmorFsPresent(PathPrefix, *Logger)
patchNode.Metadata.Labels[common.OCIHooksLabel] = ociHooksLabel

if nodeEnforcer == "none" {
patchNode.Metadata.Labels[common.SecurityFsLabel] = "no"
Expand All @@ -157,6 +167,41 @@ func snitch() {
}
}

func applyCRIOHook() error {
// TODO: hook path should be fetched from container runtime. This is the default path
hookDir := "/usr/share/containers/oci/hooks.d/"
if err := os.MkdirAll(hookDir, 0750); err != nil {
return err
}
dst, err := os.OpenFile(filepath.Join(hookDir, "ka.json"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
if err != nil {
return err
}
src, err := os.Open("/hook/ka.json")
if err != nil {
return err
}
if _, err := io.Copy(dst, src); err != nil {
return err
}
kaDir := "/usr/share/kubearmor"
if err := os.MkdirAll(kaDir, 0750); err != nil {
return err
}
dstBin, err := os.OpenFile(filepath.Join(kaDir, "hook"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0755)
if err != nil {
return err
}
srcBin, err := os.Open("/hook/hook")
if err != nil {
return err
}
if _, err := io.Copy(dstBin, srcBin); err != nil {
return err
}
return nil
}

func main() {
Execute()
}
1 change: 1 addition & 0 deletions pkg/KubeArmorOperator/common/defaults.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ var (
ApparmorFsLabel string = "kubearmor.io/apparmorfs"
SecurityFsLabel string = "kubearmor.io/securityfs"
SeccompLabel string = "kubearmor.io/seccomp"
OCIHooksLabel string = "kubearmor.io/oci-hooks"

// if any node with securityfs/lsm present
IfNodeWithSecurtiyFs bool = false
Expand Down
3 changes: 2 additions & 1 deletion pkg/KubeArmorOperator/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ require (
github.com/kubearmor/KubeArmor/KubeArmor v0.0.0-20240110164432-c2c1b121cd94
github.com/kubearmor/KubeArmor/deployments v0.0.0-20230809083125-e2d5d5709d2c
github.com/kubearmor/KubeArmor/pkg/KubeArmorController v0.0.0-20240110164432-c2c1b121cd94
github.com/opencontainers/runtime-spec v1.1.0
github.com/spf13/cobra v1.8.0
go.uber.org/zap v1.26.0
k8s.io/api v0.29.0
Expand Down Expand Up @@ -47,6 +48,7 @@ require (
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kubearmor/KubeArmor/protobuf v0.0.0-20240110164432-c2c1b121cd94 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
Expand Down Expand Up @@ -74,7 +76,6 @@ require (
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.5.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/grpc v1.63.2 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
Expand Down
9 changes: 4 additions & 5 deletions pkg/KubeArmorOperator/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kubearmor/KubeArmor/protobuf v0.0.0-20240110164432-c2c1b121cd94 h1:VZpsCDaAhuz8urwZ4OgZPGI9yh3f1HsPyAC2aniQV/4=
github.com/kubearmor/KubeArmor/protobuf v0.0.0-20240110164432-c2c1b121cd94/go.mod h1:8kx7EoCU6YHOcR4K3o6ywRwBnTv9AJZb4dyxjJkoIiQ=
github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
Expand All @@ -75,6 +77,8 @@ github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY
github.com/onsi/ginkgo/v2 v2.14.0/go.mod h1:JkUdW7JkN0V6rFvsHcJ478egV3XH9NxpD27Hal/PhZw=
github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8=
github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg=
github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
Expand Down Expand Up @@ -178,11 +182,6 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM=
google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
Expand Down
11 changes: 11 additions & 0 deletions pkg/KubeArmorOperator/hook/ka.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"version": "1.0.0",
"hook": {
"path": "/usr/share/kubearmor/hook",
"args": ["hook"]
},
"when": {
"always": true
},
"stages": ["createRuntime", "poststop"]
}
Loading

0 comments on commit 31bf733

Please sign in to comment.