Add basis for allowing the creation of configuration enforcement in gcs

This commit is the minimal set of functionality needed to allow users
to create a configuration policy that gcs can enforce.

Policy enforcement will allow users to state "only these containers, with these
command lines, etc etc" should be run. If anything in gcs doesn't match the
user supplied policy, it will end container run and report an error.

Currently, only container filesystem policy is enforced. This is done at
two points. When a pmem device is mounted, its dm-verity root hash is checked
against policy to see if it is allowed.

At the time of overlay creation, the order of layers is compared to policy to
make sure that the container is being constructed as the user expected.

Additional policy enforcement that is coming in future commits includes:

- enforce policy for scsi mounts
- enforce container command line
- enforce environment variables

.github/workflows/ci.yml

@@ -70,6 +70,7 @@ jobs:
       - run: go build ./cmd/ncproxy
       - run: go build ./cmd/dmverity-vhd
       - run: go build ./internal/tools/grantvmgroupaccess
+      - run: go build ./internal/tools/securitypolicy
       - run: go build ./internal/tools/uvmboot
       - run: go build ./internal/tools/zapdir
 

cmd/containerd-shim-runhcs-v1/pod.go

@@ -89,6 +89,7 @@ func createPod(ctx context.Context, events publisher, req *task.CreateTaskReques
 	}
 
 	var parent *uvm.UtilityVM
+	var lopts *uvm.OptionsLCOW
 	if oci.IsIsolated(s) {
 		// Create the UVM parent
 		opts, err := oci.SpecToUVMCreateOpts(ctx, s, fmt.Sprintf("%s@vm", req.ID), owner)
@@ -97,7 +98,7 @@ func createPod(ctx context.Context, events publisher, req *task.CreateTaskReques
 		}
 		switch opts.(type) {
 		case *uvm.OptionsLCOW:
-			lopts := (opts).(*uvm.OptionsLCOW)
+			lopts = (opts).(*uvm.OptionsLCOW)
 			parent, err = uvm.CreateLCOW(ctx, lopts)
 			if err != nil {
 				return nil, err
@@ -130,6 +131,13 @@ func createPod(ctx context.Context, events publisher, req *task.CreateTaskReques
 			parent.Close()
 			return nil, err
 		}
+
+		if lopts != nil {
+			err := parent.SetSecurityPolicy(ctx, lopts.SecurityPolicy)
+			if err != nil {
+				return nil, errors.Wrap(err, "unable to set security policy")
+			}
+		}
 	} else if oci.IsJobContainer(s) {
 		// If we're making a job container fake a task (i.e reuse the wcowPodSandbox logic)
 		p.sandboxTask = newWcowPodSandboxTask(ctx, events, req.ID, req.Bundle, parent, "")

go.mod

@@ -3,6 +3,7 @@ module github.com/Microsoft/hcsshim
 go 1.13
 
 require (
+	github.com/BurntSushi/toml v0.3.1
 	github.com/Microsoft/go-winio v0.4.17
 	github.com/containerd/cgroups v1.0.1
 	github.com/containerd/console v1.0.2

go.sum

@@ -34,6 +34,7 @@ github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935
 github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
 github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=

internal/guest/prot/protocol.go

@@ -8,6 +8,7 @@ import (
 	"strconv"
 
 	"github.com/Microsoft/hcsshim/internal/guest/commonutils"
+	"github.com/Microsoft/hcsshim/pkg/securitypolicy"
 	v1 "github.com/containerd/cgroups/stats/v1"
 	oci "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
@@ -519,6 +520,8 @@ const (
 	MrtVPCIDevice = ModifyResourceType("VPCIDevice")
 	// MrtContainerConstraints is the modify resource type for updating container constraints
 	MrtContainerConstraints = ModifyResourceType("ContainerConstraints")
+	// MrtSecurityPolicy is the modify resource type for updating the security policy
+	MrtSecurityPolicy = ModifyResourceType("SecurityPolicy")
 )
 
 // ModifyRequestType is the type of operation to perform on a given modify
@@ -618,6 +621,12 @@ func UnmarshalContainerModifySettings(b []byte) (*ContainerModifySettings, error
 			return &request, errors.Wrap(err, "failed to unmarshal settings as ContainerConstraintsV2")
 		}
 		msr.Settings = cc
+	case MrtSecurityPolicy:
+		policy := &securitypolicy.EncodedSecurityPolicy{}
+		if err := commonutils.UnmarshalJSONWithHresult(msrRawSettings, policy); err != nil {
+			return &request, errors.Wrap(err, "failed to unmarshal settings as EncodedSecurityPolicy")
+		}
+		msr.Settings = policy
 	default:
 		return &request, errors.Errorf("invalid ResourceType '%s'", msr.ResourceType)
 	}
@@ -713,6 +722,7 @@ type CombinedLayersV2 struct {
 	Layers            []Layer `json:",omitempty"`
 	ScratchPath       string  `json:",omitempty"`
 	ContainerRootPath string
+	ContainerId       string `json:",omitempty"`
 }
 
 // NetworkAdapter represents a network interface and its associated

internal/guest/runtime/hcsv2/uvm.go

@@ -5,6 +5,7 @@ package hcsv2
 import (
 	"bufio"
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -26,6 +27,7 @@ import (
 	"github.com/Microsoft/hcsshim/internal/guest/storage/pmem"
 	"github.com/Microsoft/hcsshim/internal/guest/storage/scsi"
 	"github.com/Microsoft/hcsshim/internal/guest/transport"
+	"github.com/Microsoft/hcsshim/pkg/securitypolicy"
 	shellwords "github.com/mattn/go-shellwords"
 	"github.com/pkg/errors"
 )
@@ -46,15 +48,60 @@ type Host struct {
 	// Rtime is the Runtime interface used by the GCS core.
 	rtime runtime.Runtime
 	vsock transport.Transport
+
+	// state required for the security policy enforcement
+	policyMutex               sync.Mutex
+	securityPolicyEnforcer    securitypolicy.SecurityPolicyEnforcer
+	securityPolicyEnforcerSet bool
 }
 
 func NewHost(rtime runtime.Runtime, vsock transport.Transport) *Host {
 	return &Host{
-		containers:        make(map[string]*Container),
-		externalProcesses: make(map[int]*externalProcess),
-		rtime:             rtime,
-		vsock:             vsock,
+		containers:                make(map[string]*Container),
+		externalProcesses:         make(map[int]*externalProcess),
+		rtime:                     rtime,
+		vsock:                     vsock,
+		securityPolicyEnforcerSet: false,
+		securityPolicyEnforcer:    &securitypolicy.OpenDoorSecurityPolicyEnforcer{},
+	}
+}
+
+// SetSecurityPolicy takes a base64 encoded security policy
+// and sets up our internal data structures we use to store
+// said policy.
+// The security policy is transmitted as json in an annotation,
+// so we first have to remove the base64 encoding that allows
+// the JSON based policy to be passed as a string. From there,
+// we decode the JSON and setup our security policy state
+func (h *Host) SetSecurityPolicy(base64_policy string) error {
+	h.policyMutex.Lock()
+	defer h.policyMutex.Unlock()
+	if h.securityPolicyEnforcerSet {
+		return errors.New("security policy has already been set")
+	}
+
+	// base64 decode the incoming policy string
+	// its base64 encoded because it is coming from an annotation
+	// annotations are a map of string to string
+	// we want to store a complex json object so.... base64 it is
+	jsonPolicy, err := base64.StdEncoding.DecodeString(base64_policy)
+	if err != nil {
+		return errors.Wrap(err, "Unable to decode policy from Base64 format")
+	}
+
+	// json unmarshall the decoded to a SecurityPolicy
+	securityPolicy := &securitypolicy.SecurityPolicy{}
+	json.Unmarshal(jsonPolicy, securityPolicy)
+
+	p, err := securitypolicy.NewSecurityPolicyEnforcer(securityPolicy)
+	if err != nil {
+		return err
 	}
+
+	h.securityPolicyEnforcer = p
+	h.securityPolicyEnforcerSet = true
+
+	return nil
 }
 
 func (h *Host) RemoveContainer(id string) {
@@ -200,9 +247,9 @@ func (h *Host) modifyHostSettings(ctx context.Context, containerID string, setti
 	case prot.MrtMappedDirectory:
 		return modifyMappedDirectory(ctx, h.vsock, settings.RequestType, settings.Settings.(*prot.MappedDirectoryV2))
 	case prot.MrtVPMemDevice:
-		return modifyMappedVPMemDevice(ctx, settings.RequestType, settings.Settings.(*prot.MappedVPMemDeviceV2))
+		return modifyMappedVPMemDevice(ctx, settings.RequestType, settings.Settings.(*prot.MappedVPMemDeviceV2), h.securityPolicyEnforcer)
 	case prot.MrtCombinedLayers:
-		return modifyCombinedLayers(ctx, settings.RequestType, settings.Settings.(*prot.CombinedLayersV2))
+		return modifyCombinedLayers(ctx, settings.RequestType, settings.Settings.(*prot.CombinedLayersV2), h.securityPolicyEnforcer)
 	case prot.MrtNetwork:
 		return modifyNetwork(ctx, settings.RequestType, settings.Settings.(*prot.NetworkAdapterV2))
 	case prot.MrtVPCIDevice:
@@ -213,6 +260,13 @@ func (h *Host) modifyHostSettings(ctx context.Context, containerID string, setti
 			return err
 		}
 		return c.modifyContainerConstraints(ctx, settings.RequestType, settings.Settings.(*prot.ContainerConstraintsV2))
+	case prot.MrtSecurityPolicy:
+		policy, ok := settings.Settings.(*securitypolicy.EncodedSecurityPolicy)
+		if !ok {
+			return errors.New("the request's settings are not of type EncodedSecurityPolicy")
+		}
+
+		return h.SetSecurityPolicy(policy.SecurityPolicy)
 	default:
 		return errors.Errorf("the ResourceType \"%s\" is not supported for UVM", settings.ResourceType)
 	}
@@ -381,12 +435,12 @@ func modifyMappedDirectory(ctx context.Context, vsock transport.Transport, rt pr
 	}
 }
 
-func modifyMappedVPMemDevice(ctx context.Context, rt prot.ModifyRequestType, vpd *prot.MappedVPMemDeviceV2) (err error) {
+func modifyMappedVPMemDevice(ctx context.Context, rt prot.ModifyRequestType, vpd *prot.MappedVPMemDeviceV2, securityPolicy securitypolicy.SecurityPolicyEnforcer) (err error) {
 	switch rt {
 	case prot.MreqtAdd:
-		return pmem.Mount(ctx, vpd.DeviceNumber, vpd.MountPath, vpd.MappingInfo, vpd.VerityInfo)
+		return pmem.Mount(ctx, vpd.DeviceNumber, vpd.MountPath, vpd.MappingInfo, vpd.VerityInfo, securityPolicy)
 	case prot.MreqtRemove:
-		return pmem.Unmount(ctx, vpd.DeviceNumber, vpd.MountPath, vpd.MappingInfo, vpd.VerityInfo)
+		return pmem.Unmount(ctx, vpd.DeviceNumber, vpd.MountPath, vpd.MappingInfo, vpd.VerityInfo, securityPolicy)
 	default:
 		return newInvalidRequestTypeError(rt)
 	}
@@ -401,7 +455,7 @@ func modifyMappedVPCIDevice(ctx context.Context, rt prot.ModifyRequestType, vpci
 	}
 }
 
-func modifyCombinedLayers(ctx context.Context, rt prot.ModifyRequestType, cl *prot.CombinedLayersV2) (err error) {
+func modifyCombinedLayers(ctx context.Context, rt prot.ModifyRequestType, cl *prot.CombinedLayersV2, securityPolicy securitypolicy.SecurityPolicyEnforcer) (err error) {
 	switch rt {
 	case prot.MreqtAdd:
 		layerPaths := make([]string, len(cl.Layers))
@@ -420,7 +474,7 @@ func modifyCombinedLayers(ctx context.Context, rt prot.ModifyRequestType, cl *pr
 			workdirPath = filepath.Join(cl.ScratchPath, "work")
 		}
 
-		return overlay.Mount(ctx, layerPaths, upperdirPath, workdirPath, cl.ContainerRootPath, readonly)
+		return overlay.Mount(ctx, layerPaths, upperdirPath, workdirPath, cl.ContainerRootPath, readonly, cl.ContainerId, securityPolicy)
 	case prot.MreqtRemove:
 		return storage.UnmountPath(ctx, cl.ContainerRootPath, true)
 	default:

internal/guest/storage/overlay/overlay.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/Microsoft/hcsshim/internal/oc"
+	"github.com/Microsoft/hcsshim/pkg/securitypolicy"
 	"github.com/pkg/errors"
 	"go.opencensus.io/trace"
 	"golang.org/x/sys/unix"
@@ -30,11 +31,15 @@ var (
 //
 // Always creates `rootfsPath`. On mount failure the created `rootfsPath` will
 // be automatically cleaned up.
-func Mount(ctx context.Context, layerPaths []string, upperdirPath, workdirPath, rootfsPath string, readonly bool) (err error) {
+func Mount(ctx context.Context, layerPaths []string, upperdirPath, workdirPath, rootfsPath string, readonly bool, containerId string, securityPolicy securitypolicy.SecurityPolicyEnforcer) (err error) {
 	_, span := trace.StartSpan(ctx, "overlay::Mount")
 	defer span.End()
 	defer func() { oc.SetSpanStatus(span, err) }()
 
+	if err := securityPolicy.EnforceOverlayMountPolicy(containerId, layerPaths); err != nil {
+		return err
+	}
+
 	lowerdir := strings.Join(layerPaths, ":")
 	span.AddAttributes(
 		trace.StringAttribute("layerPaths", lowerdir),

internal/guest/storage/overlay/overlay_test.go

@@ -7,6 +7,13 @@ import (
 	"errors"
 	"os"
 	"testing"
+
+	"github.com/Microsoft/hcsshim/internal/guest/storage/test/policy"
+	"github.com/Microsoft/hcsshim/pkg/securitypolicy"
+)
+
+const (
+	fakeContainerId = "1"
 )
 
 type undo struct {
@@ -76,7 +83,7 @@ func Test_Mount_Success(t *testing.T) {
 		return nil
 	}
 
-	err := Mount(context.Background(), []string{"/layer1", "/layer2"}, "/upper", "/work", "/root", false)
+	err := Mount(context.Background(), []string{"/layer1", "/layer2"}, "/upper", "/work", "/root", false, fakeContainerId, openDoorSecurityPolicyEnforcer())
 	if err != nil {
 		t.Fatalf("expected no error got: %v", err)
 	}
@@ -120,11 +127,80 @@ func Test_Mount_Readonly_Success(t *testing.T) {
 		return nil
 	}
 
-	err := Mount(context.Background(), []string{"/layer1", "/layer2"}, "", "", "/root", false)
+	err := Mount(context.Background(), []string{"/layer1", "/layer2"}, "", "", "/root", false, fakeContainerId, openDoorSecurityPolicyEnforcer())
 	if err != nil {
 		t.Fatalf("expected no error got: %v", err)
 	}
 	if !rootCreated {
 		t.Fatal("expected root to be created")
 	}
 }
+
+func Test_Security_Policy_Enforcement(t *testing.T) {
+	undo := captureTestMethods()
+	defer undo.Close()
+
+	var upperCreated, workCreated, rootCreated bool
+	osMkdirAll = func(path string, perm os.FileMode) error {
+		if perm != 0755 {
+			t.Errorf("os.MkdirAll at: %s, perm: %v expected perm: 0755", path, perm)
+		}
+		switch path {
+		case "/upper":
+			upperCreated = true
+			return nil
+		case "/work":
+			workCreated = true
+			return nil
+		case "/root":
+			rootCreated = true
+			return nil
+		}
+		return errors.New("unexpected os.MkdirAll path")
+	}
+	unixMount = func(source string, target string, fstype string, flags uintptr, data string) error {
+		if source != "overlay" {
+			t.Errorf("expected source: 'overlay' got: %v", source)
+		}
+		if target != "/root" {
+			t.Errorf("expected target: '/root' got: %v", target)
+		}
+		if fstype != "overlay" {
+			t.Errorf("expected fstype: 'overlay' got: %v", fstype)
+		}
+		if flags != 0 {
+			t.Errorf("expected flags: '0' got: %v", flags)
+		}
+		if data != "lowerdir=/layer1:/layer2,upperdir=/upper,workdir=/work" {
+			t.Errorf("expected data: 'lowerdir=/layer1:/layer2,upperdir=/upper,workdir=/work' got: %v", data)
+		}
+		return nil
+	}
+
+	enforcer := mountMonitoringSecurityPolicyEnforcer()
+	err := Mount(context.Background(), []string{"/layer1", "/layer2"}, "/upper", "/work", "/root", false, fakeContainerId, enforcer)
+	if err != nil {
+		t.Fatalf("expected no error got: %v", err)
+	}
+	if !upperCreated || !workCreated || !rootCreated {
+		t.Fatalf("expected all upper: %v, work: %v, root: %v to be created", upperCreated, workCreated, rootCreated)
+	}
+
+	expectedPmem := 0
+	if enforcer.PmemMountCalls != expectedPmem {
+		t.Errorf("expected %d attempt at pmem mount enforcement, got %d", expectedPmem, enforcer.PmemMountCalls)
+	}
+
+	expectedOverlay := 1
+	if enforcer.OverlayMountCalls != expectedOverlay {
+		t.Fatalf("expected %d attempts at overlay mount enforcement, got %d", expectedOverlay, enforcer.OverlayMountCalls)
+	}
+}
+
+func openDoorSecurityPolicyEnforcer() securitypolicy.SecurityPolicyEnforcer {
+	return &securitypolicy.OpenDoorSecurityPolicyEnforcer{}
+}
+
+func mountMonitoringSecurityPolicyEnforcer() *policy.MountMonitoringSecurityPolicyEnforcer {
+	return &policy.MountMonitoringSecurityPolicyEnforcer{}
+}