VMCache: the core and the client

VMCache is a new function that creates VMs as caches before using it.
It helps speed up new container creation.
The function consists of a server and some clients communicating
through Unix socket.  The protocol is gRPC in protocols/cache/cache.proto.
The VMCache server will create some VMs and cache them by factory cache.
It will convert the VM to gRPC format and transport it when gets
requestion from clients.
Factory grpccache is the VMCache client.  It will request gRPC format
VM and convert it back to a VM.  If VMCache function is enabled,
kata-runtime will request VM from factory grpccache when it creates
a new sandbox.

VMCache has two options.
vm_cache_number specifies the number of caches of VMCache:
unspecified or == 0   --> VMCache is disabled
> 0                   --> will be set to the specified number
vm_cache_endpoint specifies the address of the Unix socket.

This commit just includes the core and the client of VMCache.

Currently, VM cache still cannot work with VM templating and vsock.
And just support qemu.

Fixes: kata-containers#52

Signed-off-by: Hui Zhu <[email protected]>

cli/config/configuration-qemu.toml.in

@@ -219,6 +219,30 @@ enable_iothreads = @DEFENABLEIOTHREADS@
 # Default false
 #enable_template = true
 
+# The number of caches of VMCache:
+# unspecified or == 0   --> VMCache is disabled
+# > 0                   --> will be set to the specified number
+#
+# VMCache is a function that creates VMs as caches before using it.
+# It helps speed up new container creation.
+# The function consists of a server and some clients communicating
+# through Unix socket.  The protocol is gRPC in protocols/cache/cache.proto.
+# The VMCache server will create some VMs and cache them by factory cache.
+# It will convert the VM to gRPC format and transport it when gets
+# requestion from clients.
+# Factory grpccache is the VMCache client.  It will request gRPC format
+# VM and convert it back to a VM.  If VMCache function is enabled,
+# kata-runtime will request VM from factory grpccache when it creates
+# a new sandbox.
+#
+# Default 0
+#vm_cache_number = 0
+
+# Specify the address of the Unix socket that is used by VMCache.
+#
+# Default /var/run/kata-containers/cache.sock
+#vm_cache_endpoint = "/var/run/kata-containers/cache.sock"
+
 [proxy.@PROJECT_TYPE@]
 path = "@PROXYPATH@"
 

hack/update-generated-runtime-proto.sh

@@ -0,0 +1,14 @@
+#
+# Copyright 2019 HyperHQ Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+protoc \
+	-I=$GOPATH/src \
+	-I=$GOPATH/src/github.com/gogo/protobuf/protobuf \
+	--proto_path=protocols/cache \
+	--gogofast_out=\
+Mgoogle/protobuf/empty.proto=github.com/gogo/protobuf/types,\
+plugins=grpc:protocols/cache \
+	protocols/cache/cache.proto

pkg/katautils/config-settings.go

@@ -43,6 +43,8 @@ const defaultHotplugVFIOOnRootBus bool = false
 const defaultEntropySource = "/dev/urandom"
 const defaultGuestHookPath string = ""
 
+const defaultVMCacheEndpoint string = "/var/run/kata-containers/cache.sock"
+
 // Default config file used by stateless systems.
 var defaultRuntimeConfiguration = "/usr/share/defaults/kata-containers/configuration.toml"
 

pkg/katautils/config.go

@@ -77,7 +77,9 @@ type tomlConfig struct {
 }
 
 type factory struct {
-	Template bool `toml:"enable_template"`
+	Template        bool   `toml:"enable_template"`
+	VMCacheNumber   uint   `toml:"vm_cache_number"`
+	VMCacheEndpoint string `toml:"vm_cache_endpoint"`
 }
 
 type hypervisor struct {
@@ -544,7 +546,14 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 }
 
 func newFactoryConfig(f factory) (oci.FactoryConfig, error) {
-	return oci.FactoryConfig{Template: f.Template}, nil
+	if f.VMCacheEndpoint == "" {
+		f.VMCacheEndpoint = defaultVMCacheEndpoint
+	}
+	return oci.FactoryConfig{
+		Template:        f.Template,
+		VMCacheNumber:   f.VMCacheNumber,
+		VMCacheEndpoint: f.VMCacheEndpoint,
+	}, nil
 }
 
 func newShimConfig(s shim) (vc.ShimConfig, error) {
@@ -910,6 +919,10 @@ func checkNetNsConfig(config oci.RuntimeConfig) error {
 
 // checkFactoryConfig ensures the VM factory configuration is valid.
 func checkFactoryConfig(config oci.RuntimeConfig) error {
+	if config.FactoryConfig.Template && config.FactoryConfig.VMCacheNumber > 0 {
+		return errors.New("VM factory cannot work together with VM cache")
+	}
+
 	if config.FactoryConfig.Template {
 		if config.HypervisorConfig.InitrdPath == "" {
 			return errors.New("Factory option enable_template requires an initrd image")
@@ -920,6 +933,18 @@ func checkFactoryConfig(config oci.RuntimeConfig) error {
 		}
 	}
 
+	if config.FactoryConfig.VMCacheNumber > 0 {
+		if config.HypervisorType != vc.QemuHypervisor {
+			return errors.New("VM cache just support qemu")
+		}
+		if config.AgentType != vc.KataContainersAgent {
+			return errors.New("VM cache just support kata agent")
+		}
+		if config.HypervisorConfig.UseVSock {
+			return errors.New("config vsock conflicts with VM cache, please disable one of them")
+		}
+	}
+
 	return nil
 }
 

pkg/katautils/config_test.go

@@ -184,6 +184,10 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 		Enable: false,
 	}
 
+	factoryConfig := oci.FactoryConfig{
+		VMCacheEndpoint: defaultVMCacheEndpoint,
+	}
+
 	runtimeConfig := oci.RuntimeConfig{
 		HypervisorType:   defaultHypervisor,
 		HypervisorConfig: hypervisorConfig,
@@ -199,6 +203,8 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 
 		NetmonConfig:    netmonConfig,
 		DisableNewNetNs: disableNewNetNs,
+
+		FactoryConfig: factoryConfig,
 	}
 
 	err = SetKernelParams(&runtimeConfig)
@@ -626,6 +632,10 @@ func TestMinimalRuntimeConfig(t *testing.T) {
 		Enable: false,
 	}
 
+	expectedFactoryConfig := oci.FactoryConfig{
+		VMCacheEndpoint: defaultVMCacheEndpoint,
+	}
+
 	expectedConfig := oci.RuntimeConfig{
 		HypervisorType:   defaultHypervisor,
 		HypervisorConfig: expectedHypervisorConfig,
@@ -640,6 +650,8 @@ func TestMinimalRuntimeConfig(t *testing.T) {
 		ShimConfig: expectedShimConfig,
 
 		NetmonConfig: expectedNetmonConfig,
+
+		FactoryConfig: expectedFactoryConfig,
 	}
 	err = SetKernelParams(&expectedConfig)
 	if err != nil {
@@ -1376,7 +1388,8 @@ func TestUpdateRuntimeConfigurationFactoryConfig(t *testing.T) {
 
 	config := oci.RuntimeConfig{}
 	expectedFactoryConfig := oci.FactoryConfig{
-		Template: true,
+		Template:        true,
+		VMCacheEndpoint: defaultVMCacheEndpoint,
 	}
 
 	tomlConf := tomlConfig{Factory: factory{Template: true}}

pkg/katautils/create.go

@@ -53,19 +53,25 @@ func needSystemd(config vc.HypervisorConfig) bool {
 
 // HandleFactory  set the factory
 func HandleFactory(ctx context.Context, vci vc.VC, runtimeConfig *oci.RuntimeConfig) {
-	if !runtimeConfig.FactoryConfig.Template {
+	if !runtimeConfig.FactoryConfig.Template && runtimeConfig.FactoryConfig.VMCacheNumber == 0 {
 		return
 	}
 
 	factoryConfig := vf.Config{
-		Template: true,
+		Template:        runtimeConfig.FactoryConfig.Template,
+		VMCache:         runtimeConfig.FactoryConfig.VMCacheNumber > 0,
+		VMCacheEndpoint: runtimeConfig.FactoryConfig.VMCacheEndpoint,
 		VMConfig: vc.VMConfig{
 			HypervisorType:   runtimeConfig.HypervisorType,
 			HypervisorConfig: runtimeConfig.HypervisorConfig,
 			AgentType:        runtimeConfig.AgentType,
 			AgentConfig:      runtimeConfig.AgentConfig,
 		},
 	}
+	if runtimeConfig.FactoryConfig.VMCacheNumber > 0 {
+		factoryConfig.VMConfig.ProxyType = runtimeConfig.ProxyType
+		factoryConfig.VMConfig.ProxyConfig = runtimeConfig.ProxyConfig
+	}
 
 	kataUtilsLogger.WithField("factory", factoryConfig).Info("load vm factory")