VMCache: the new function that creates VMs as caches before using it

VM cache helps speeding up new container creation.
To use it,  need set option "enable_vm_cache" to true and use
"kata-runtime cache" command start the VM cache server that created
some VMs as VM cache.  Then each kata-runtime will request VM from
VM cache server.

Currently, VM cache still cannot work with VM templating and vsock.
And just support qemu.

Fixes: kata-containers#52

Signed-off-by: Hui Zhu <[email protected]>

cli/cache.go

@@ -0,0 +1,177 @@
+// Copyright (c) 2019 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"net"
+	"os"
+	"os/signal"
+	"path/filepath"
+	rDebug "runtime/debug"
+
+	google_protobuf "github.com/golang/protobuf/ptypes/empty"
+	pb "github.com/kata-containers/runtime/protocols/cache"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	vf "github.com/kata-containers/runtime/virtcontainers/factory"
+	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli"
+	"golang.org/x/sys/unix"
+	"google.golang.org/grpc"
+)
+
+type cacheServer struct {
+	rpc     *grpc.Server
+	factory vc.Factory
+}
+
+var jsonVMConfig *pb.GrpcVMConfig
+
+func (s *cacheServer) Config(ctx context.Context, empty *google_protobuf.Empty) (*pb.GrpcVMConfig, error) {
+	if jsonVMConfig == nil {
+		config := s.factory.Config()
+
+		var err error
+		jsonVMConfig, err = config.ToGrpc()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return jsonVMConfig, nil
+}
+
+func (s *cacheServer) GetBaseVM(ctx context.Context, empty *google_protobuf.Empty) (*pb.GrpcVM, error) {
+	config := s.factory.Config()
+
+	vm, err := s.factory.GetBaseVM(ctx, config)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to GetBaseVM")
+	}
+
+	return vm.ToGrpc(config)
+}
+
+func getUnixListener(path string) (net.Listener, error) {
+	err := os.MkdirAll(filepath.Dir(path), 0755)
+	if err != nil {
+		return nil, err
+	}
+	if err = unix.Unlink(path); err != nil && !os.IsNotExist(err) {
+		return nil, err
+	}
+	l, err := net.Listen("unix", path)
+	if err != nil {
+		return nil, err
+	}
+	if err = os.Chmod(path, 0660); err != nil {
+		l.Close()
+		return nil, err
+	}
+	return l, nil
+}
+
+var handledSignals = []os.Signal{
+	unix.SIGTERM,
+	unix.SIGINT,
+	unix.SIGUSR1,
+	unix.SIGPIPE,
+}
+
+func handleSignals(s *cacheServer, signals chan os.Signal) chan struct{} {
+	done := make(chan struct{}, 1)
+	go func() {
+		for {
+			sig := <-signals
+			kataLog.WithField("signal", sig).Debug("received signal")
+			switch sig {
+			case unix.SIGUSR1:
+				kataLog.WithField("stack", rDebug.Stack()).Debug("dump stack")
+			case unix.SIGPIPE:
+				continue
+			default:
+				s.rpc.GracefulStop()
+				close(done)
+				return
+			}
+		}
+	}()
+	return done
+}
+
+var cacheCLICommand = cli.Command{
+	Name:  "cache",
+	Usage: "run a vm cache server",
+	Flags: []cli.Flag{
+		cli.UintFlag{
+			Name:  "number, n",
+			Value: 1,
+			Usage: `number of cache`,
+		},
+	},
+	Action: func(context *cli.Context) error {
+		cacheNum := context.Uint("number")
+		if cacheNum == 0 {
+			return errors.New("number of cache must big than 0")
+		}
+		ctx, err := cliContextToContext(context)
+		if err != nil {
+			return err
+		}
+		runtimeConfig, ok := context.App.Metadata["runtimeConfig"].(oci.RuntimeConfig)
+		if !ok {
+			return errors.New("invalid runtime config")
+		}
+		if !runtimeConfig.FactoryConfig.VMCache {
+			return errors.New("vm cache not enabled")
+		}
+
+		factoryConfig := vf.Config{
+			Template: runtimeConfig.FactoryConfig.Template,
+			Cache:    cacheNum,
+			VMCache:  true,
+			VMConfig: vc.VMConfig{
+				HypervisorType:   runtimeConfig.HypervisorType,
+				HypervisorConfig: runtimeConfig.HypervisorConfig,
+				AgentType:        runtimeConfig.AgentType,
+				AgentConfig:      runtimeConfig.AgentConfig,
+				ProxyType:        runtimeConfig.ProxyType,
+				ProxyConfig:      runtimeConfig.ProxyConfig,
+			},
+		}
+		f, err := vf.NewFactory(ctx, factoryConfig, false)
+		if err != nil {
+			return err
+		}
+		defer f.CloseFactory(ctx)
+
+		s := &cacheServer{
+			rpc:     grpc.NewServer(),
+			factory: f,
+		}
+		pb.RegisterCacheServiceServer(s.rpc, s)
+
+		l, err := getUnixListener(runtimeConfig.FactoryConfig.VMCacheEndpoint)
+		if err != nil {
+			return err
+		}
+		defer l.Close()
+
+		signals := make(chan os.Signal, 2048)
+		done := handleSignals(s, signals)
+		signal.Notify(signals, handledSignals...)
+
+		kataLog.WithField("endpoint", runtimeConfig.FactoryConfig.VMCacheEndpoint).Info("VM cache server start")
+		s.rpc.Serve(l)
+
+		<-done
+
+		kataLog.WithField("endpoint", runtimeConfig.FactoryConfig.VMCacheEndpoint).Info("VM cache server stop")
+
+		return nil
+	},
+}

cli/config/configuration-qemu.toml.in

@@ -219,6 +219,18 @@ enable_iothreads = @DEFENABLEIOTHREADS@
 # Default false
 #enable_template = true
 
+# VM cache support.  Once enabled, need use "kata-runtime cache" command
+# start the VM cache server that created some VMs as VM cache.  And each
+# kata-runtime will request VM from VM cache server.  It helps speeding
+# up new container creation.
+#
+# Default false
+#enable_vm_cache = true
+
+# Specify the endpoint of transport VM from the VM cache server to runtime.
+# Default /var/run/kata-containers/cache.sock
+#vm_cache_endpoint = "/var/run/kata-containers/cache.sock"
+
 [proxy.@PROJECT_TYPE@]
 path = "@PROXYPATH@"
 

cli/main.go

@@ -134,6 +134,7 @@ var runtimeCommands = []cli.Command{
 	kataEnvCLICommand,
 	kataNetworkCLICommand,
 	factoryCLICommand,
+	cacheCLICommand,
 }
 
 // runtimeBeforeSubcommands is the function to run before command-line

pkg/katautils/config-settings.go

@@ -43,6 +43,8 @@ const defaultHotplugVFIOOnRootBus bool = false
 const defaultEntropySource = "/dev/urandom"
 const defaultGuestHookPath string = ""
 
+const defaultVMCacheEndpoint string = "/var/run/kata-containers/cache.sock"
+
 // Default config file used by stateless systems.
 var defaultRuntimeConfiguration = "/usr/share/defaults/kata-containers/configuration.toml"
 

pkg/katautils/config.go

@@ -77,7 +77,9 @@ type tomlConfig struct {
 }
 
 type factory struct {
-	Template bool `toml:"enable_template"`
+	Template        bool   `toml:"enable_template"`
+	VMCache         bool   `toml:"enable_vm_cache"`
+	VMCacheEndpoint string `toml:"vm_cache_endpoint"`
 }
 
 type hypervisor struct {
@@ -548,7 +550,14 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 }
 
 func newFactoryConfig(f factory) (oci.FactoryConfig, error) {
-	return oci.FactoryConfig{Template: f.Template}, nil
+	if f.VMCacheEndpoint == "" {
+		f.VMCacheEndpoint = defaultVMCacheEndpoint
+	}
+	return oci.FactoryConfig{
+		Template:        f.Template,
+		VMCache:         f.VMCache,
+		VMCacheEndpoint: f.VMCacheEndpoint,
+	}, nil
 }
 
 func newShimConfig(s shim) (vc.ShimConfig, error) {
@@ -912,6 +921,10 @@ func checkNetNsConfig(config oci.RuntimeConfig) error {
 
 // checkFactoryConfig ensures the VM factory configuration is valid.
 func checkFactoryConfig(config oci.RuntimeConfig) error {
+	if config.FactoryConfig.Template && config.FactoryConfig.VMCache {
+		return errors.New("VM factory cannot work together with VM cache")
+	}
+
 	if config.FactoryConfig.Template {
 		if config.HypervisorConfig.InitrdPath == "" {
 			return errors.New("Factory option enable_template requires an initrd image")
@@ -922,6 +935,18 @@ func checkFactoryConfig(config oci.RuntimeConfig) error {
 		}
 	}
 
+	if config.FactoryConfig.VMCache {
+		if config.HypervisorType != vc.QemuHypervisor {
+			return errors.New("VM cache just support qemu")
+		}
+		if config.AgentType != vc.KataContainersAgent {
+			return errors.New("VM cache just support kata agent")
+		}
+		if config.HypervisorConfig.UseVSock {
+			return errors.New("config vsock conflicts with VM cache, please disable one of them")
+		}
+	}
+
 	return nil
 }
 

pkg/katautils/create.go

@@ -53,19 +53,25 @@ func needSystemd(config vc.HypervisorConfig) bool {
 
 // HandleFactory  set the factory
 func HandleFactory(ctx context.Context, vci vc.VC, runtimeConfig *oci.RuntimeConfig) {
-	if !runtimeConfig.FactoryConfig.Template {
+	if !runtimeConfig.FactoryConfig.Template && !runtimeConfig.FactoryConfig.VMCache {
 		return
 	}
 
 	factoryConfig := vf.Config{
-		Template: true,
+		Template:        runtimeConfig.FactoryConfig.Template,
+		VMCache:         runtimeConfig.FactoryConfig.VMCache,
+		VMCacheEndpoint: runtimeConfig.FactoryConfig.VMCacheEndpoint,
 		VMConfig: vc.VMConfig{
 			HypervisorType:   runtimeConfig.HypervisorType,
 			HypervisorConfig: runtimeConfig.HypervisorConfig,
 			AgentType:        runtimeConfig.AgentType,
 			AgentConfig:      runtimeConfig.AgentConfig,
 		},
 	}
+	if runtimeConfig.FactoryConfig.VMCache {
+		factoryConfig.VMConfig.ProxyType = runtimeConfig.ProxyType
+		factoryConfig.VMConfig.ProxyConfig = runtimeConfig.ProxyConfig
+	}
 
 	kataUtilsLogger.WithField("factory", factoryConfig).Info("load vm factory")