From 48df99c873cde032bedcae89767c314591bfa72b Mon Sep 17 00:00:00 2001
From: Hui Zhu <teawater@hyper.sh>
Date: Thu, 20 Dec 2018 21:49:38 +0800
Subject: [PATCH] VMCache: the new function that creates VMs as caches before
 using it

VM cache helps speeding up new container creation.
To use it,  need set option "enable_vm_cache" to true and use
"kata-vmcache" command start the VM cache server that created
some VMs as VM cache.  Then each kata-runtime will request VM from
VM cache server.

Currently, VM cache still cannot work with VM templating and vsock.
And just support qemu.

Fixes: #52

Signed-off-by: Hui Zhu <teawater@hyper.sh>
---
 .gitignore                                    |   2 +
 Makefile                                      |  65 ++++-
 cli/config/configuration-qemu.toml.in         |  15 +
 cli/kata-env.go                               |   8 +-
 cli/kata-env_test.go                          |  10 +-
 cli/kata-vmcache/main.go                      | 205 ++++++++++++++
 cli/main.go                                   |  25 +-
 cli/main_test.go                              |   6 +-
 cli/utils.go                                  |   8 +-
 pkg/katautils/config-settings.go              |   2 +
 pkg/katautils/config.go                       |  29 +-
 pkg/katautils/config_test.go                  |  15 +-
 pkg/katautils/create.go                       |  10 +-
 pkg/katautils/utils.go                        |  26 ++
 protocols/cache/cache.pb.go                   | 257 ++++++++++++++++++
 protocols/cache/cache.proto                   |  35 +++
 virtcontainers/agent.go                       |   6 +
 virtcontainers/factory.go                     |   6 +
 virtcontainers/factory/factory.go             |  90 ++----
 virtcontainers/factory/factory_test.go        |  38 +--
 virtcontainers/factory/grpcCache/grpcCache.go |  62 +++++
 virtcontainers/fc.go                          |   9 +
 virtcontainers/hyperstart_agent.go            |  10 +
 virtcontainers/hypervisor.go                  |   2 +
 virtcontainers/kata_agent.go                  |  28 +-
 virtcontainers/kata_agent.go.rej              | 116 ++++++++
 virtcontainers/mock_hypervisor.go             |   9 +
 virtcontainers/noop_agent.go                  |   7 +
 virtcontainers/pkg/oci/utils.go               |   6 +
 virtcontainers/qemu.go                        |  53 ++++
 virtcontainers/qemu_test.go                   |  19 ++
 virtcontainers/utils/compare.go               |  74 +++++
 virtcontainers/vm.go                          | 115 ++++++++
 virtcontainers/vm_test.go                     |  20 ++
 34 files changed, 1245 insertions(+), 143 deletions(-)
 create mode 100644 cli/kata-vmcache/main.go
 create mode 100644 protocols/cache/cache.pb.go
 create mode 100644 protocols/cache/cache.proto
 create mode 100644 virtcontainers/factory/grpcCache/grpcCache.go
 create mode 100644 virtcontainers/kata_agent.go.rej
 create mode 100644 virtcontainers/utils/compare.go

diff --git a/.gitignore b/.gitignore
index 7b6eee072d..fc7a28d4fd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,11 +4,13 @@
 /cli/config/configuration-fc.toml
 /cli/config/configuration-qemu.toml
 /cli/config-generated.go
+/cli/kata-vmcache/config-generated.go
 /cli/coverage.html
 /containerd-shim-kata-v2
 /data/kata-collect-data.sh
 /kata-netmon
 /kata-runtime
+/kata-vmcache
 /virtcontainers/hack/virtc/virtc
 /virtcontainers/hook/mock/hook
 /virtcontainers/profile.cov
diff --git a/Makefile b/Makefile
index 51e5fd0f97..78de58aa40 100644
--- a/Makefile
+++ b/Makefile
@@ -183,6 +183,10 @@ SHIMV2 = containerd-shim-kata-v2
 SHIMV2_OUTPUT = $(CURDIR)/$(SHIMV2)
 SHIMV2_DIR = $(CLI_DIR)/$(SHIMV2)
 
+VMCACHE = kata-vmcache
+VMCACHE_OUTPUT = $(CURDIR)/$(VMCACHE)
+VMCACHE_DIR = $(CLI_DIR)/$(VMCACHE)
+
 SOURCES := $(shell find . 2>&1 | grep -E '.*\.(c|h|go)$$')
 VERSION := ${shell cat ./VERSION}
 COMMIT_NO := $(shell git rev-parse HEAD 2> /dev/null || true)
@@ -357,7 +361,7 @@ define SHOW_ARCH
   $(shell printf "\\t%s%s\\\n" "$(1)" $(if $(filter $(ARCH),$(1))," (default)",""))
 endef
 
-all: runtime containerd-shim-v2 netmon
+all: runtime containerd-shim-v2 netmon $(VMCACHE)
 
 containerd-shim-v2: $(SHIMV2_OUTPUT)
 
@@ -366,6 +370,11 @@ netmon: $(NETMON_TARGET_OUTPUT)
 $(NETMON_TARGET_OUTPUT): $(SOURCES)
 	$(QUIET_BUILD)(cd $(NETMON_DIR) && go build $(BUILDFLAGS) -o $@ -ldflags "-X main.version=$(VERSION)")
 
+$(VMCACHE): $(VMCACHE_OUTPUT)
+
+$(VMCACHE_OUTPUT): $(TARGET_OUTPUT)
+	$(QUIET_BUILD)(cd $(VMCACHE_DIR)/ && go build -i -o $@ .)
+
 runtime: $(TARGET_OUTPUT) $(CONFIGS)
 .DEFAULT: default
 
@@ -420,6 +429,45 @@ endef
 
 export GENERATED_CODE
 
+define VMCACHE_GENERATED_CODE
+// WARNING: This file is auto-generated - DO NOT EDIT!
+//
+// Note that some variables are "var" to allow them to be modified
+// by the tests.
+package main
+
+import (
+	"fmt"
+)
+
+// name is the name of the runtime
+const name = "$(VMCACHE)"
+
+// name of the project
+const project = "$(PROJECT_NAME)"
+
+// prefix used to denote non-standard CLI commands and options.
+const projectPrefix = "$(PROJECT_TYPE)"
+
+// commit is the git commit the runtime is compiled from.
+var commit = "$(COMMIT)"
+
+// version is the runtime version.
+var version = "$(VERSION)"
+
+// project-specific option names
+var configFilePathOption = fmt.Sprintf("%s-config", projectPrefix)
+
+// Default config file used by stateless systems.
+var defaultRuntimeConfiguration = "$(CONFIG_PATH)"
+
+// Alternate config file that takes precedence over
+// defaultRuntimeConfiguration.
+var defaultSysConfRuntimeConfiguration = "$(SYSCONFIG)"
+endef
+
+export VMCACHE_GENERATED_CODE
+
 #Install an executable file
 # params:
 # $1 : file to install
@@ -443,12 +491,16 @@ $(if $(findstring uncompressed,$1),vmlinux.container,vmlinuz.container)
 endef
 
 GENERATED_CONFIG = $(CLI_DIR)/config-generated.go
+VMCACHE_GENERATED_CONFIG = $(VMCACHE_DIR)/config-generated.go
 
-GENERATED_GO_FILES += $(GENERATED_CONFIG)
+GENERATED_GO_FILES += $(GENERATED_CONFIG) $(VMCACHE_GENERATED_CONFIG)
 
 $(GENERATED_CONFIG): Makefile VERSION
 	$(QUIET_GENERATE)echo "$$GENERATED_CODE" >$@
 
+$(VMCACHE_GENERATED_CONFIG): Makefile VERSION
+	$(QUIET_GENERATE)echo "$$VMCACHE_GENERATED_CODE" >$@
+
 $(TARGET_OUTPUT): $(EXTRA_DEPS) $(SOURCES) $(GENERATED_GO_FILES) $(GENERATED_FILES) Makefile | show-summary
 	$(QUIET_BUILD)(cd $(CLI_DIR) && go build $(BUILDFLAGS) -o $@ .)
 
@@ -539,7 +591,7 @@ check-go-static:
 coverage:
 	$(QUIET_TEST).ci/go-test.sh html-coverage
 
-install: default runtime install-scripts install-completions install-configs install-bin install-containerd-shim-v2 install-bin-libexec
+install: default runtime install-scripts install-completions install-configs install-bin install-containerd-shim-v2 install-kata-vmcache install-bin-libexec
 
 install-bin: $(BINLIST)
 	$(QUIET_INST)$(foreach f,$(BINLIST),$(call INSTALL_EXEC,$f,$(BINDIR)))
@@ -547,6 +599,9 @@ install-bin: $(BINLIST)
 install-containerd-shim-v2: $(SHIMV2)
 	$(QUIET_INST)$(call INSTALL_EXEC,$<,$(BINDIR))
 
+install-kata-vmcache: $(VMCACHE)
+	$(QUIET_INST)$(call INSTALL_EXEC,$<,$(BINDIR))
+
 install-bin-libexec: $(BINLIBEXECLIST)
 	$(QUIET_INST)$(foreach f,$(BINLIBEXECLIST),$(call INSTALL_EXEC,$f,$(PKGLIBEXECDIR)))
 
@@ -561,7 +616,7 @@ install-completions:
 	$(QUIET_INST)install --mode 0644 -D  $(BASH_COMPLETIONS) $(DESTDIR)/$(BASH_COMPLETIONSDIR)/$(notdir $(BASH_COMPLETIONS));
 
 clean:
-	$(QUIET_CLEAN)rm -f $(TARGET) $(SHIMV2) $(NETMON_TARGET) $(CONFIGS) $(GENERATED_GO_FILES) $(GENERATED_FILES) $(COLLECT_SCRIPT)
+	$(QUIET_CLEAN)rm -f $(TARGET) $(SHIMV2) $(VMCACHE_OUTPUT) $(NETMON_TARGET) $(CONFIGS) $(GENERATED_GO_FILES) $(GENERATED_FILES) $(COLLECT_SCRIPT)
 
 show-usage: show-header
 	@printf "• Overview:\n"
@@ -635,6 +690,8 @@ endif
           "$(foreach b,$(sort $(BINLIST)),$(shell printf "\\t - $(shell readlink -m $(DESTDIR)/$(BINDIR)/$(b))\\\n"))"
 	@printf \
           "$(foreach b,$(sort $(SHIMV2)),$(shell printf "\\t - $(shell readlink -m $(DESTDIR)/$(BINDIR)/$(b))\\\n"))"
+	@printf \
+          "$(foreach b,$(sort $(VMCACHE)),$(shell printf "\\t - $(shell readlink -m $(DESTDIR)/$(BINDIR)/$(b))\\\n"))"
 	@printf \
           "$(foreach b,$(sort $(BINLIBEXECLIST)),$(shell printf "\\t - $(shell readlink -m $(DESTDIR)/$(PKGLIBEXECDIR)/$(b))\\\n"))"
 	@printf \
diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
index 4be52f4a94..26e6fe0f0f 100644
--- a/cli/config/configuration-qemu.toml.in
+++ b/cli/config/configuration-qemu.toml.in
@@ -219,6 +219,21 @@ enable_iothreads = @DEFENABLEIOTHREADS@
 # Default false
 #enable_template = true
 
+# VM cache support.  Once enabled, need use "kata-vmcache" command start
+# the VM cache server that created some VMs (number is set by option number)
+# as VM cache.  Each kata-runtime will request VM from VM cache server
+# through vm_cache_endpoint.
+# It helps speeding up new container creation.
+#
+# Default false
+#enable_vm_cache = true
+
+# Specify the endpoint of transport VM from the VM cache server to runtime.
+# "kata-vmcache" will start a gRPC server in this endpoint.
+# Each kata-runtime will request VM through gRPC protocol.
+# Default /var/run/kata-containers/cache.sock
+#vm_cache_endpoint = "/var/run/kata-containers/cache.sock"
+
 [proxy.@PROJECT_TYPE@]
 path = "@PROXYPATH@"
 
diff --git a/cli/kata-env.go b/cli/kata-env.go
index 8bf2d05760..12f90a36e3 100644
--- a/cli/kata-env.go
+++ b/cli/kata-env.go
@@ -247,7 +247,7 @@ func getProxyInfo(config oci.RuntimeConfig) (ProxyInfo, error) {
 	proxyConfig := config.ProxyConfig
 	version, err := getCommandVersion(proxyConfig.Path)
 	if err != nil {
-		version = unknown
+		version = katautils.Unknown
 	}
 
 	proxy := ProxyInfo{
@@ -265,7 +265,7 @@ func getNetmonInfo(config oci.RuntimeConfig) (NetmonInfo, error) {
 
 	version, err := getCommandVersion(netmonConfig.Path)
 	if err != nil {
-		version = unknown
+		version = katautils.Unknown
 	}
 
 	netmon := NetmonInfo{
@@ -292,7 +292,7 @@ func getShimInfo(config oci.RuntimeConfig) (ShimInfo, error) {
 
 	version, err := getCommandVersion(shimPath)
 	if err != nil {
-		version = unknown
+		version = katautils.Unknown
 	}
 
 	shim := ShimInfo{
@@ -318,7 +318,7 @@ func getHypervisorInfo(config oci.RuntimeConfig) HypervisorInfo {
 
 	version, err := getCommandVersion(hypervisorPath)
 	if err != nil {
-		version = unknown
+		version = katautils.Unknown
 	}
 
 	return HypervisorInfo{
diff --git a/cli/kata-env_test.go b/cli/kata-env_test.go
index a3b6a5970a..8d70254e49 100644
--- a/cli/kata-env_test.go
+++ b/cli/kata-env_test.go
@@ -543,7 +543,7 @@ func TestEnvGetEnvInfoNoHypervisorVersion(t *testing.T) {
 	err = os.Remove(config.HypervisorConfig.HypervisorPath)
 	assert.NoError(err)
 
-	expectedEnv.Hypervisor.Version = unknown
+	expectedEnv.Hypervisor.Version = katautils.Unknown
 
 	env, err := getEnvInfo(configFile, config)
 	assert.NoError(err)
@@ -696,7 +696,7 @@ func TestEnvGetProxyInfoNoVersion(t *testing.T) {
 	err = os.Remove(config.ProxyConfig.Path)
 	assert.NoError(t, err)
 
-	expectedProxy.Version = unknown
+	expectedProxy.Version = katautils.Unknown
 
 	proxy, err := getProxyInfo(config)
 	assert.NoError(t, err)
@@ -740,7 +740,7 @@ func TestEnvGetNetmonInfoNoVersion(t *testing.T) {
 	err = os.Remove(config.NetmonConfig.Path)
 	assert.NoError(t, err)
 
-	expectedNetmon.Version = unknown
+	expectedNetmon.Version = katautils.Unknown
 
 	netmon, err := getNetmonInfo(config)
 	assert.NoError(t, err)
@@ -787,7 +787,7 @@ func TestEnvGetShimInfoNoVersion(t *testing.T) {
 	exit 1`)
 	assert.NoError(t, err)
 
-	expectedShim.Version = unknown
+	expectedShim.Version = katautils.Unknown
 
 	shim, err := getShimInfo(config)
 	assert.NoError(t, err)
@@ -1227,5 +1227,5 @@ func TestGetHypervisorInfo(t *testing.T) {
 	assert.NoError(err)
 
 	info = getHypervisorInfo(config)
-	assert.Equal(info.Version, unknown)
+	assert.Equal(info.Version, katautils.Unknown)
 }
diff --git a/cli/kata-vmcache/main.go b/cli/kata-vmcache/main.go
new file mode 100644
index 0000000000..7d3ae1588c
--- /dev/null
+++ b/cli/kata-vmcache/main.go
@@ -0,0 +1,205 @@
+// Copyright (c) 2019 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"os/signal"
+	"path/filepath"
+	rDebug "runtime/debug"
+
+	google_protobuf "github.com/golang/protobuf/ptypes/empty"
+	"github.com/kata-containers/runtime/pkg/katautils"
+	pb "github.com/kata-containers/runtime/protocols/cache"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	vf "github.com/kata-containers/runtime/virtcontainers/factory"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+	"golang.org/x/sys/unix"
+	"google.golang.org/grpc"
+)
+
+var usage = fmt.Sprintf(`%s kata VM cache server
+
+When enable_vm_cache enabled, %s start the VM cache server
+that created some VMs (number is set by option number) as
+VM cache.
+Each kata-runtime will request VM from VM cache server
+through vm_cache_endpoint.
+It helps speeding up new container creation.`, name, name)
+
+var kataLog = logrus.New()
+
+type cacheServer struct {
+	rpc     *grpc.Server
+	factory vc.Factory
+}
+
+var jsonVMConfig *pb.GrpcVMConfig
+
+// Config requests base factory config and convert it to gRPC protocol.
+func (s *cacheServer) Config(ctx context.Context, empty *google_protobuf.Empty) (*pb.GrpcVMConfig, error) {
+	if jsonVMConfig == nil {
+		config := s.factory.Config()
+
+		var err error
+		jsonVMConfig, err = config.ToGrpc()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return jsonVMConfig, nil
+}
+
+// GetBaseVM requests a paused VM and convert it to gRPC protocol.
+func (s *cacheServer) GetBaseVM(ctx context.Context, empty *google_protobuf.Empty) (*pb.GrpcVM, error) {
+	config := s.factory.Config()
+
+	vm, err := s.factory.GetBaseVM(ctx, config)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to GetBaseVM")
+	}
+
+	return vm.ToGrpc(config)
+}
+
+func getUnixListener(path string) (net.Listener, error) {
+	err := os.MkdirAll(filepath.Dir(path), 0755)
+	if err != nil {
+		return nil, err
+	}
+	if err = unix.Unlink(path); err != nil && !os.IsNotExist(err) {
+		return nil, err
+	}
+	l, err := net.Listen("unix", path)
+	if err != nil {
+		return nil, err
+	}
+	if err = os.Chmod(path, 0660); err != nil {
+		l.Close()
+		return nil, err
+	}
+	return l, nil
+}
+
+var handledSignals = []os.Signal{
+	unix.SIGTERM,
+	unix.SIGINT,
+	unix.SIGUSR1,
+	unix.SIGPIPE,
+}
+
+func handleSignals(s *cacheServer, signals chan os.Signal) chan struct{} {
+	done := make(chan struct{}, 1)
+	go func() {
+		for {
+			sig := <-signals
+			kataLog.WithField("signal", sig).Debug("received signal")
+			switch sig {
+			case unix.SIGUSR1:
+				kataLog.WithField("stack", rDebug.Stack()).Debug("dump stack")
+			case unix.SIGPIPE:
+				continue
+			default:
+				s.rpc.GracefulStop()
+				close(done)
+				return
+			}
+		}
+	}()
+	return done
+}
+
+func main() {
+	app := cli.NewApp()
+	app.Name = name
+	app.Usage = usage
+	app.Writer = os.Stdout
+	app.Version = katautils.MakeVersionString(name, version, commit, specs.Version)
+	app.Flags = []cli.Flag{
+		cli.StringFlag{
+			Name:  configFilePathOption,
+			Usage: project + " config file path",
+		},
+		cli.UintFlag{
+			Name:  "number, n",
+			Value: 1,
+			Usage: "number of cache",
+		},
+	}
+	app.Action = func(c *cli.Context) error {
+		cacheNum := c.Uint("number")
+		if cacheNum == 0 {
+			return errors.New("cache number must be greater than zero")
+		}
+
+		ctx := context.Background()
+
+		katautils.SetConfigOptions(name, defaultRuntimeConfiguration, defaultSysConfRuntimeConfiguration)
+		_, runtimeConfig, err := katautils.LoadConfiguration(c.GlobalString(configFilePathOption), false, false)
+		if err != nil {
+			return errors.Wrap(err, "invalid runtime config")
+		}
+		if !runtimeConfig.FactoryConfig.VMCache {
+			return errors.New("vm cache not enabled")
+		}
+
+		factoryConfig := vf.Config{
+			Template: runtimeConfig.FactoryConfig.Template,
+			Cache:    cacheNum,
+			VMCache:  true,
+			VMConfig: vc.VMConfig{
+				HypervisorType:   runtimeConfig.HypervisorType,
+				HypervisorConfig: runtimeConfig.HypervisorConfig,
+				AgentType:        runtimeConfig.AgentType,
+				AgentConfig:      runtimeConfig.AgentConfig,
+				ProxyType:        runtimeConfig.ProxyType,
+				ProxyConfig:      runtimeConfig.ProxyConfig,
+			},
+		}
+		f, err := vf.NewFactory(ctx, factoryConfig, false)
+		if err != nil {
+			return err
+		}
+		defer f.CloseFactory(ctx)
+
+		s := &cacheServer{
+			rpc:     grpc.NewServer(),
+			factory: f,
+		}
+		pb.RegisterCacheServiceServer(s.rpc, s)
+
+		l, err := getUnixListener(runtimeConfig.FactoryConfig.VMCacheEndpoint)
+		if err != nil {
+			return err
+		}
+		defer l.Close()
+
+		signals := make(chan os.Signal, 2048)
+		done := handleSignals(s, signals)
+		signal.Notify(signals, handledSignals...)
+
+		kataLog.WithField("endpoint", runtimeConfig.FactoryConfig.VMCacheEndpoint).Info("VM cache server start")
+		s.rpc.Serve(l)
+
+		<-done
+
+		kataLog.WithField("endpoint", runtimeConfig.FactoryConfig.VMCacheEndpoint).Info("VM cache server stop")
+
+		return nil
+	}
+
+	err := app.Run(os.Args)
+	if err != nil {
+		kataLog.Fatal(err)
+	}
+}
diff --git a/cli/main.go b/cli/main.go
index a021a0a775..657a219233 100644
--- a/cli/main.go
+++ b/cli/main.go
@@ -409,30 +409,7 @@ func commandNotFound(c *cli.Context, command string) {
 // makeVersionString returns a multi-line string describing the runtime
 // version along with the version of the OCI specification it supports.
 func makeVersionString() string {
-	v := make([]string, 0, 3)
-
-	versionStr := version
-	if versionStr == "" {
-		versionStr = unknown
-	}
-
-	v = append(v, name+"  : "+versionStr)
-
-	commitStr := commit
-	if commitStr == "" {
-		commitStr = unknown
-	}
-
-	v = append(v, "   commit   : "+commitStr)
-
-	specVersionStr := specs.Version
-	if specVersionStr == "" {
-		specVersionStr = unknown
-	}
-
-	v = append(v, "   OCI specs: "+specVersionStr)
-
-	return strings.Join(v, "\n")
+	return katautils.MakeVersionString(name, version, commit, specs.Version)
 }
 
 // setCLIGlobals modifies various cli package global variables
diff --git a/cli/main_test.go b/cli/main_test.go
index f9a65fce81..6bed46fb85 100644
--- a/cli/main_test.go
+++ b/cli/main_test.go
@@ -867,7 +867,7 @@ func TestMainMakeVersionStringNoVersion(t *testing.T) {
 
 	v := makeVersionString()
 
-	testVersionString(assert, v, unknown, commit, specs.Version)
+	testVersionString(assert, v, katautils.Unknown, commit, specs.Version)
 }
 
 func TestMainMakeVersionStringNoCommit(t *testing.T) {
@@ -882,7 +882,7 @@ func TestMainMakeVersionStringNoCommit(t *testing.T) {
 
 	v := makeVersionString()
 
-	testVersionString(assert, v, version, unknown, specs.Version)
+	testVersionString(assert, v, version, katautils.Unknown, specs.Version)
 }
 
 func TestMainMakeVersionStringNoOCIVersion(t *testing.T) {
@@ -897,7 +897,7 @@ func TestMainMakeVersionStringNoOCIVersion(t *testing.T) {
 
 	v := makeVersionString()
 
-	testVersionString(assert, v, version, commit, unknown)
+	testVersionString(assert, v, version, commit, katautils.Unknown)
 }
 
 func TestMainCreateRuntimeApp(t *testing.T) {
diff --git a/cli/utils.go b/cli/utils.go
index cc996ca3a1..14e533c9d6 100644
--- a/cli/utils.go
+++ b/cli/utils.go
@@ -13,10 +13,6 @@ import (
 	"github.com/kata-containers/runtime/pkg/katautils"
 )
 
-const (
-	unknown = "<<unknown>>"
-)
-
 // variables to allow tests to modify the values
 var (
 	procVersion = "/proc/version"
@@ -79,11 +75,11 @@ func getDistroDetails() (name, version string, err error) {
 	}
 
 	if name == "" {
-		name = unknown
+		name = katautils.Unknown
 	}
 
 	if version == "" {
-		version = unknown
+		version = katautils.Unknown
 	}
 
 	return name, version, nil
diff --git a/pkg/katautils/config-settings.go b/pkg/katautils/config-settings.go
index b85efcb5f3..da7ce7ba85 100644
--- a/pkg/katautils/config-settings.go
+++ b/pkg/katautils/config-settings.go
@@ -43,6 +43,8 @@ const defaultHotplugVFIOOnRootBus bool = false
 const defaultEntropySource = "/dev/urandom"
 const defaultGuestHookPath string = ""
 
+const defaultVMCacheEndpoint string = "/var/run/kata-containers/cache.sock"
+
 // Default config file used by stateless systems.
 var defaultRuntimeConfiguration = "/usr/share/defaults/kata-containers/configuration.toml"
 
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
index b286ef1fe2..dcef33a09e 100644
--- a/pkg/katautils/config.go
+++ b/pkg/katautils/config.go
@@ -77,7 +77,9 @@ type tomlConfig struct {
 }
 
 type factory struct {
-	Template bool `toml:"enable_template"`
+	Template        bool   `toml:"enable_template"`
+	VMCache         bool   `toml:"enable_vm_cache"`
+	VMCacheEndpoint string `toml:"vm_cache_endpoint"`
 }
 
 type hypervisor struct {
@@ -544,7 +546,14 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 }
 
 func newFactoryConfig(f factory) (oci.FactoryConfig, error) {
-	return oci.FactoryConfig{Template: f.Template}, nil
+	if f.VMCacheEndpoint == "" {
+		f.VMCacheEndpoint = defaultVMCacheEndpoint
+	}
+	return oci.FactoryConfig{
+		Template:        f.Template,
+		VMCache:         f.VMCache,
+		VMCacheEndpoint: f.VMCacheEndpoint,
+	}, nil
 }
 
 func newShimConfig(s shim) (vc.ShimConfig, error) {
@@ -908,6 +917,10 @@ func checkNetNsConfig(config oci.RuntimeConfig) error {
 
 // checkFactoryConfig ensures the VM factory configuration is valid.
 func checkFactoryConfig(config oci.RuntimeConfig) error {
+	if config.FactoryConfig.Template && config.FactoryConfig.VMCache {
+		return errors.New("VM factory cannot work together with VM cache")
+	}
+
 	if config.FactoryConfig.Template {
 		if config.HypervisorConfig.InitrdPath == "" {
 			return errors.New("Factory option enable_template requires an initrd image")
@@ -918,6 +931,18 @@ func checkFactoryConfig(config oci.RuntimeConfig) error {
 		}
 	}
 
+	if config.FactoryConfig.VMCache {
+		if config.HypervisorType != vc.QemuHypervisor {
+			return errors.New("VM cache just support qemu")
+		}
+		if config.AgentType != vc.KataContainersAgent {
+			return errors.New("VM cache just support kata agent")
+		}
+		if config.HypervisorConfig.UseVSock {
+			return errors.New("config vsock conflicts with VM cache, please disable one of them")
+		}
+	}
+
 	return nil
 }
 
diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go
index 12bac15979..2743531d0e 100644
--- a/pkg/katautils/config_test.go
+++ b/pkg/katautils/config_test.go
@@ -184,6 +184,10 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 		Enable: false,
 	}
 
+	factoryConfig := oci.FactoryConfig{
+		VMCacheEndpoint: defaultVMCacheEndpoint,
+	}
+
 	runtimeConfig := oci.RuntimeConfig{
 		HypervisorType:   defaultHypervisor,
 		HypervisorConfig: hypervisorConfig,
@@ -199,6 +203,8 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 
 		NetmonConfig:    netmonConfig,
 		DisableNewNetNs: disableNewNetNs,
+
+		FactoryConfig: factoryConfig,
 	}
 
 	err = SetKernelParams(&runtimeConfig)
@@ -626,6 +632,10 @@ func TestMinimalRuntimeConfig(t *testing.T) {
 		Enable: false,
 	}
 
+	expectedFactoryConfig := oci.FactoryConfig{
+		VMCacheEndpoint: defaultVMCacheEndpoint,
+	}
+
 	expectedConfig := oci.RuntimeConfig{
 		HypervisorType:   defaultHypervisor,
 		HypervisorConfig: expectedHypervisorConfig,
@@ -640,6 +650,8 @@ func TestMinimalRuntimeConfig(t *testing.T) {
 		ShimConfig: expectedShimConfig,
 
 		NetmonConfig: expectedNetmonConfig,
+
+		FactoryConfig: expectedFactoryConfig,
 	}
 	err = SetKernelParams(&expectedConfig)
 	if err != nil {
@@ -1376,7 +1388,8 @@ func TestUpdateRuntimeConfigurationFactoryConfig(t *testing.T) {
 
 	config := oci.RuntimeConfig{}
 	expectedFactoryConfig := oci.FactoryConfig{
-		Template: true,
+		Template:        true,
+		VMCacheEndpoint: defaultVMCacheEndpoint,
 	}
 
 	tomlConf := tomlConfig{Factory: factory{Template: true}}
diff --git a/pkg/katautils/create.go b/pkg/katautils/create.go
index 0e07e8003c..af576ae989 100644
--- a/pkg/katautils/create.go
+++ b/pkg/katautils/create.go
@@ -53,12 +53,14 @@ func needSystemd(config vc.HypervisorConfig) bool {
 
 // HandleFactory  set the factory
 func HandleFactory(ctx context.Context, vci vc.VC, runtimeConfig *oci.RuntimeConfig) {
-	if !runtimeConfig.FactoryConfig.Template {
+	if !runtimeConfig.FactoryConfig.Template && !runtimeConfig.FactoryConfig.VMCache {
 		return
 	}
 
 	factoryConfig := vf.Config{
-		Template: true,
+		Template:        runtimeConfig.FactoryConfig.Template,
+		VMCache:         runtimeConfig.FactoryConfig.VMCache,
+		VMCacheEndpoint: runtimeConfig.FactoryConfig.VMCacheEndpoint,
 		VMConfig: vc.VMConfig{
 			HypervisorType:   runtimeConfig.HypervisorType,
 			HypervisorConfig: runtimeConfig.HypervisorConfig,
@@ -66,6 +68,10 @@ func HandleFactory(ctx context.Context, vci vc.VC, runtimeConfig *oci.RuntimeCon
 			AgentConfig:      runtimeConfig.AgentConfig,
 		},
 	}
+	if runtimeConfig.FactoryConfig.VMCache {
+		factoryConfig.VMConfig.ProxyType = runtimeConfig.ProxyType
+		factoryConfig.VMConfig.ProxyConfig = runtimeConfig.ProxyConfig
+	}
 
 	kataUtilsLogger.WithField("factory", factoryConfig).Info("load vm factory")
 
diff --git a/pkg/katautils/utils.go b/pkg/katautils/utils.go
index a8c1ca96f4..a2b17969db 100644
--- a/pkg/katautils/utils.go
+++ b/pkg/katautils/utils.go
@@ -18,6 +18,9 @@ import (
 
 const (
 	k8sEmptyDir = "kubernetes.io~empty-dir"
+
+	// Unknown is set to the unknown item of the kata version string.
+	Unknown = "<<unknown>>"
 )
 
 // FileExists test is a file exiting or not
@@ -134,3 +137,26 @@ func RunCommandFull(args []string, includeStderr bool) (string, error) {
 func RunCommand(args []string) (string, error) {
 	return RunCommandFull(args, false)
 }
+
+// MakeVersionString returns a multi-line string describing the kata
+// version.
+func MakeVersionString(nameStr, versionStr, commitStr, specVersionStr string) string {
+	v := make([]string, 0, 3)
+
+	if versionStr == "" {
+		versionStr = Unknown
+	}
+	v = append(v, nameStr+"  : "+versionStr)
+
+	if commitStr == "" {
+		commitStr = Unknown
+	}
+	v = append(v, "   commit   : "+commitStr)
+
+	if specVersionStr == "" {
+		specVersionStr = Unknown
+	}
+	v = append(v, "   OCI specs: "+specVersionStr)
+
+	return strings.Join(v, "\n")
+}
diff --git a/protocols/cache/cache.pb.go b/protocols/cache/cache.pb.go
new file mode 100644
index 0000000000..9361945619
--- /dev/null
+++ b/protocols/cache/cache.pb.go
@@ -0,0 +1,257 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// source: cache.proto
+
+/*
+Package cache is a generated protocol buffer package.
+
+It is generated from these files:
+	cache.proto
+
+It has these top-level messages:
+	GrpcVMConfig
+	GrpcVM
+*/
+package cache
+
+import proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+import google_protobuf "github.com/golang/protobuf/ptypes/empty"
+
+import (
+	context "golang.org/x/net/context"
+	grpc "google.golang.org/grpc"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
+
+type GrpcVMConfig struct {
+	Data        []byte `protobuf:"bytes,1,opt,name=Data,proto3" json:"Data,omitempty"`
+	AgentConfig []byte `protobuf:"bytes,2,opt,name=AgentConfig,proto3" json:"AgentConfig,omitempty"`
+}
+
+func (m *GrpcVMConfig) Reset()                    { *m = GrpcVMConfig{} }
+func (m *GrpcVMConfig) String() string            { return proto.CompactTextString(m) }
+func (*GrpcVMConfig) ProtoMessage()               {}
+func (*GrpcVMConfig) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
+
+func (m *GrpcVMConfig) GetData() []byte {
+	if m != nil {
+		return m.Data
+	}
+	return nil
+}
+
+func (m *GrpcVMConfig) GetAgentConfig() []byte {
+	if m != nil {
+		return m.AgentConfig
+	}
+	return nil
+}
+
+type GrpcVM struct {
+	Id         string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
+	Hypervisor []byte `protobuf:"bytes,2,opt,name=hypervisor,proto3" json:"hypervisor,omitempty"`
+	ProxyPid   int64  `protobuf:"varint,3,opt,name=proxyPid" json:"proxyPid,omitempty"`
+	ProxyURL   string `protobuf:"bytes,4,opt,name=proxyURL" json:"proxyURL,omitempty"`
+	Cpu        uint32 `protobuf:"varint,5,opt,name=cpu" json:"cpu,omitempty"`
+	Memory     uint32 `protobuf:"varint,6,opt,name=memory" json:"memory,omitempty"`
+	CpuDelta   uint32 `protobuf:"varint,7,opt,name=cpuDelta" json:"cpuDelta,omitempty"`
+}
+
+func (m *GrpcVM) Reset()                    { *m = GrpcVM{} }
+func (m *GrpcVM) String() string            { return proto.CompactTextString(m) }
+func (*GrpcVM) ProtoMessage()               {}
+func (*GrpcVM) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
+
+func (m *GrpcVM) GetId() string {
+	if m != nil {
+		return m.Id
+	}
+	return ""
+}
+
+func (m *GrpcVM) GetHypervisor() []byte {
+	if m != nil {
+		return m.Hypervisor
+	}
+	return nil
+}
+
+func (m *GrpcVM) GetProxyPid() int64 {
+	if m != nil {
+		return m.ProxyPid
+	}
+	return 0
+}
+
+func (m *GrpcVM) GetProxyURL() string {
+	if m != nil {
+		return m.ProxyURL
+	}
+	return ""
+}
+
+func (m *GrpcVM) GetCpu() uint32 {
+	if m != nil {
+		return m.Cpu
+	}
+	return 0
+}
+
+func (m *GrpcVM) GetMemory() uint32 {
+	if m != nil {
+		return m.Memory
+	}
+	return 0
+}
+
+func (m *GrpcVM) GetCpuDelta() uint32 {
+	if m != nil {
+		return m.CpuDelta
+	}
+	return 0
+}
+
+func init() {
+	proto.RegisterType((*GrpcVMConfig)(nil), "cache.GrpcVMConfig")
+	proto.RegisterType((*GrpcVM)(nil), "cache.GrpcVM")
+}
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConn
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion4
+
+// Client API for CacheService service
+
+type CacheServiceClient interface {
+	Config(ctx context.Context, in *google_protobuf.Empty, opts ...grpc.CallOption) (*GrpcVMConfig, error)
+	GetBaseVM(ctx context.Context, in *google_protobuf.Empty, opts ...grpc.CallOption) (*GrpcVM, error)
+}
+
+type cacheServiceClient struct {
+	cc *grpc.ClientConn
+}
+
+func NewCacheServiceClient(cc *grpc.ClientConn) CacheServiceClient {
+	return &cacheServiceClient{cc}
+}
+
+func (c *cacheServiceClient) Config(ctx context.Context, in *google_protobuf.Empty, opts ...grpc.CallOption) (*GrpcVMConfig, error) {
+	out := new(GrpcVMConfig)
+	err := grpc.Invoke(ctx, "/cache.CacheService/Config", in, out, c.cc, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *cacheServiceClient) GetBaseVM(ctx context.Context, in *google_protobuf.Empty, opts ...grpc.CallOption) (*GrpcVM, error) {
+	out := new(GrpcVM)
+	err := grpc.Invoke(ctx, "/cache.CacheService/GetBaseVM", in, out, c.cc, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// Server API for CacheService service
+
+type CacheServiceServer interface {
+	Config(context.Context, *google_protobuf.Empty) (*GrpcVMConfig, error)
+	GetBaseVM(context.Context, *google_protobuf.Empty) (*GrpcVM, error)
+}
+
+func RegisterCacheServiceServer(s *grpc.Server, srv CacheServiceServer) {
+	s.RegisterService(&_CacheService_serviceDesc, srv)
+}
+
+func _CacheService_Config_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(google_protobuf.Empty)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(CacheServiceServer).Config(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/cache.CacheService/Config",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(CacheServiceServer).Config(ctx, req.(*google_protobuf.Empty))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _CacheService_GetBaseVM_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(google_protobuf.Empty)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(CacheServiceServer).GetBaseVM(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/cache.CacheService/GetBaseVM",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(CacheServiceServer).GetBaseVM(ctx, req.(*google_protobuf.Empty))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _CacheService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "cache.CacheService",
+	HandlerType: (*CacheServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "Config",
+			Handler:    _CacheService_Config_Handler,
+		},
+		{
+			MethodName: "GetBaseVM",
+			Handler:    _CacheService_GetBaseVM_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "cache.proto",
+}
+
+func init() { proto.RegisterFile("cache.proto", fileDescriptor0) }
+
+var fileDescriptor0 = []byte{
+	// 281 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x90, 0x41, 0x4b, 0xc3, 0x40,
+	0x14, 0x84, 0x49, 0xd2, 0x46, 0xfb, 0x9a, 0x8a, 0x3c, 0xa1, 0x2c, 0x11, 0x24, 0xf4, 0x94, 0x53,
+	0x0a, 0x15, 0x7f, 0x80, 0x36, 0xd2, 0x8b, 0x05, 0x89, 0xd8, 0x7b, 0xba, 0x79, 0x4d, 0x03, 0x4d,
+	0x77, 0xd9, 0x6e, 0xc4, 0xfc, 0x31, 0x7f, 0x9f, 0x64, 0x13, 0x43, 0x7b, 0xf0, 0xb6, 0x33, 0xf3,
+	0x66, 0x58, 0x3e, 0x18, 0xf3, 0x94, 0xef, 0x29, 0x92, 0x4a, 0x68, 0x81, 0x43, 0x23, 0xfc, 0xfb,
+	0x5c, 0x88, 0xfc, 0x40, 0x73, 0x63, 0x6e, 0xab, 0xdd, 0x9c, 0x4a, 0xa9, 0xeb, 0xf6, 0x66, 0x16,
+	0x83, 0xb7, 0x52, 0x92, 0x6f, 0xd6, 0x4b, 0x71, 0xdc, 0x15, 0x39, 0x22, 0x0c, 0xe2, 0x54, 0xa7,
+	0xcc, 0x0a, 0xac, 0xd0, 0x4b, 0xcc, 0x1b, 0x03, 0x18, 0x3f, 0xe7, 0x74, 0xd4, 0xed, 0x09, 0xb3,
+	0x4d, 0x74, 0x6e, 0xcd, 0x7e, 0x2c, 0x70, 0xdb, 0x19, 0xbc, 0x01, 0xbb, 0xc8, 0x4c, 0x7d, 0x94,
+	0xd8, 0x45, 0x86, 0x0f, 0x00, 0xfb, 0x5a, 0x92, 0xfa, 0x2a, 0x4e, 0x42, 0x75, 0xdd, 0x33, 0x07,
+	0x7d, 0xb8, 0x96, 0x4a, 0x7c, 0xd7, 0xef, 0x45, 0xc6, 0x9c, 0xc0, 0x0a, 0x9d, 0xa4, 0xd7, 0x7d,
+	0xf6, 0x99, 0xbc, 0xb1, 0x81, 0x59, 0xec, 0x35, 0xde, 0x82, 0xc3, 0x65, 0xc5, 0x86, 0x81, 0x15,
+	0x4e, 0x92, 0xe6, 0x89, 0x53, 0x70, 0x4b, 0x2a, 0x85, 0xaa, 0x99, 0x6b, 0xcc, 0x4e, 0x35, 0x2b,
+	0x5c, 0x56, 0x31, 0x1d, 0x74, 0xca, 0xae, 0x4c, 0xd2, 0xeb, 0x45, 0x0d, 0xde, 0xb2, 0x81, 0xf4,
+	0xd1, 0x7c, 0x87, 0x13, 0x3e, 0x81, 0xdb, 0x81, 0x98, 0x46, 0x2d, 0xb6, 0xe8, 0x0f, 0x5b, 0xf4,
+	0xda, 0x60, 0xf3, 0xef, 0xa2, 0x16, 0xf1, 0x05, 0xb5, 0x05, 0x8c, 0x56, 0xa4, 0x5f, 0xd2, 0x13,
+	0x6d, 0xd6, 0xff, 0x36, 0x27, 0x17, 0xcd, 0xad, 0x6b, 0xe2, 0xc7, 0xdf, 0x00, 0x00, 0x00, 0xff,
+	0xff, 0xae, 0x34, 0x14, 0x00, 0xb3, 0x01, 0x00, 0x00,
+}
diff --git a/protocols/cache/cache.proto b/protocols/cache/cache.proto
new file mode 100644
index 0000000000..6eada1eb02
--- /dev/null
+++ b/protocols/cache/cache.proto
@@ -0,0 +1,35 @@
+//
+// Copyright 2019 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+syntax = "proto3";
+
+package cache;
+
+import "google/protobuf/empty.proto";
+
+service CacheService {
+    rpc Config(google.protobuf.Empty) returns (GrpcVMConfig);
+    rpc GetBaseVM(google.protobuf.Empty) returns (GrpcVM);
+}
+
+message GrpcVMConfig {
+    bytes Data = 1;
+    bytes AgentConfig = 2;
+}
+
+message GrpcVM {
+    string id = 1;
+
+    bytes hypervisor = 2;
+
+    int64 proxyPid = 3;
+    string proxyURL = 4;
+
+    uint32 cpu = 5;
+    uint32 memory = 6;
+
+    uint32 cpuDelta = 7;
+}
diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go
index 9cfce0f349..804f47b789 100644
--- a/virtcontainers/agent.go
+++ b/virtcontainers/agent.go
@@ -151,6 +151,9 @@ type agent interface {
 	// set to use an existing proxy
 	setProxy(sandbox *Sandbox, proxy proxy, pid int, url string) error
 
+	// set to use an existing proxy from Grpc
+	setProxyFromGrpc(proxy proxy, pid int, url string)
+
 	// get agent url
 	getAgentURL() (string, error)
 
@@ -225,6 +228,9 @@ type agent interface {
 	// configure will update agent settings based on provided arguments
 	configure(h hypervisor, id, sharePath string, builtin bool, config interface{}) error
 
+	// configureFromGrpc will update agent settings based on provided arguments which from Grpc
+	configureFromGrpc(id string, builtin bool, config interface{}) error
+
 	// getVMPath will return the agent vm socket's directory path
 	getVMPath(id string) string
 
diff --git a/virtcontainers/factory.go b/virtcontainers/factory.go
index 45306a5f0a..8579d8a975 100644
--- a/virtcontainers/factory.go
+++ b/virtcontainers/factory.go
@@ -9,9 +9,15 @@ import "context"
 
 // Factory controls how a new VM is created.
 type Factory interface {
+	// Config returns base factory config.
+	Config() VMConfig
+
 	// GetVM gets a new VM from the factory.
 	GetVM(ctx context.Context, config VMConfig) (*VM, error)
 
+	// GetBaseVM returns a paused VM created by the base factory.
+	GetBaseVM(ctx context.Context, config VMConfig) (*VM, error)
+
 	// CloseFactory closes and cleans up the factory.
 	CloseFactory(ctx context.Context)
 }
diff --git a/virtcontainers/factory/factory.go b/virtcontainers/factory/factory.go
index e11e1632c6..78599f54f7 100644
--- a/virtcontainers/factory/factory.go
+++ b/virtcontainers/factory/factory.go
@@ -8,13 +8,14 @@ package factory
 import (
 	"context"
 	"fmt"
-	"reflect"
 
 	vc "github.com/kata-containers/runtime/virtcontainers"
 	"github.com/kata-containers/runtime/virtcontainers/factory/base"
 	"github.com/kata-containers/runtime/virtcontainers/factory/cache"
 	"github.com/kata-containers/runtime/virtcontainers/factory/direct"
+	"github.com/kata-containers/runtime/virtcontainers/factory/grpcCache"
 	"github.com/kata-containers/runtime/virtcontainers/factory/template"
+	"github.com/kata-containers/runtime/virtcontainers/utils"
 	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/sirupsen/logrus"
 )
@@ -24,7 +25,10 @@ var factoryLogger = logrus.FieldLogger(logrus.New())
 // Config is a collection of VM factory configurations.
 type Config struct {
 	Template bool
-	Cache    uint
+
+	VMCache         bool
+	Cache           uint
+	VMCacheEndpoint string
 
 	VMConfig vc.VMConfig
 }
@@ -65,6 +69,11 @@ func NewFactory(ctx context.Context, config Config, fetchOnly bool) (vc.Factory,
 		} else {
 			b = template.New(ctx, config.VMConfig)
 		}
+	} else if config.VMCache && config.Cache == 0 {
+		b, err = grpcCache.New(ctx, config.VMCacheEndpoint)
+		if err != nil {
+			return nil, err
+		}
 	} else {
 		b = direct.New(ctx, config.VMConfig)
 	}
@@ -100,71 +109,6 @@ func resetHypervisorConfig(config *vc.VMConfig) {
 	config.ProxyConfig = vc.ProxyConfig{}
 }
 
-func compareStruct(foo, bar reflect.Value) bool {
-	for i := 0; i < foo.NumField(); i++ {
-		if !deepCompareValue(foo.Field(i), bar.Field(i)) {
-			return false
-		}
-	}
-
-	return true
-}
-
-func compareMap(foo, bar reflect.Value) bool {
-	if foo.Len() != bar.Len() {
-		return false
-	}
-
-	for _, k := range foo.MapKeys() {
-		if !deepCompareValue(foo.MapIndex(k), bar.MapIndex(k)) {
-			return false
-		}
-	}
-
-	return true
-}
-
-func compareSlice(foo, bar reflect.Value) bool {
-	if foo.Len() != bar.Len() {
-		return false
-	}
-	for j := 0; j < foo.Len(); j++ {
-		if !deepCompareValue(foo.Index(j), bar.Index(j)) {
-			return false
-		}
-	}
-	return true
-}
-
-func deepCompareValue(foo, bar reflect.Value) bool {
-	if !foo.IsValid() || !bar.IsValid() {
-		return foo.IsValid() == bar.IsValid()
-	}
-
-	if foo.Type() != bar.Type() {
-		return false
-	}
-	switch foo.Kind() {
-	case reflect.Map:
-		return compareMap(foo, bar)
-	case reflect.Array:
-		fallthrough
-	case reflect.Slice:
-		return compareSlice(foo, bar)
-	case reflect.Struct:
-		return compareStruct(foo, bar)
-	default:
-		return foo.Interface() == bar.Interface()
-	}
-}
-
-func deepCompare(foo, bar interface{}) bool {
-	v1 := reflect.ValueOf(foo)
-	v2 := reflect.ValueOf(bar)
-
-	return deepCompareValue(v1, v2)
-}
-
 // It's important that baseConfig and newConfig are passed by value!
 func checkVMConfig(config1, config2 vc.VMConfig) error {
 	if config1.HypervisorType != config2.HypervisorType {
@@ -179,7 +123,7 @@ func checkVMConfig(config1, config2 vc.VMConfig) error {
 	resetHypervisorConfig(&config1)
 	resetHypervisorConfig(&config2)
 
-	if !deepCompare(config1, config2) {
+	if !utils.DeepCompare(config1, config2) {
 		return fmt.Errorf("hypervisor config does not match, base: %+v. new: %+v", config1, config2)
 	}
 
@@ -282,6 +226,16 @@ func (f *factory) GetVM(ctx context.Context, config vc.VMConfig) (*vc.VM, error)
 	return vm, nil
 }
 
+// Config returns base factory config.
+func (f *factory) Config() vc.VMConfig {
+	return f.base.Config()
+}
+
+// GetBaseVM returns a paused VM created by the base factory.
+func (f *factory) GetBaseVM(ctx context.Context, config vc.VMConfig) (*vc.VM, error) {
+	return f.base.GetBaseVM(ctx, config)
+}
+
 // CloseFactory closes the factory.
 func (f *factory) CloseFactory(ctx context.Context) {
 	f.base.CloseFactory(ctx)
diff --git a/virtcontainers/factory/factory_test.go b/virtcontainers/factory/factory_test.go
index deaa02c20e..a8e08606c2 100644
--- a/virtcontainers/factory/factory_test.go
+++ b/virtcontainers/factory/factory_test.go
@@ -10,11 +10,11 @@ import (
 	"io/ioutil"
 	"testing"
 
-	"github.com/sirupsen/logrus"
-	"github.com/stretchr/testify/assert"
-
 	vc "github.com/kata-containers/runtime/virtcontainers"
 	"github.com/kata-containers/runtime/virtcontainers/factory/base"
+	"github.com/kata-containers/runtime/virtcontainers/utils"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestNewFactory(t *testing.T) {
@@ -279,41 +279,41 @@ func TestDeepCompare(t *testing.T) {
 
 	foo := vc.VMConfig{}
 	bar := vc.VMConfig{}
-	assert.True(deepCompare(foo, bar))
+	assert.True(utils.DeepCompare(foo, bar))
 
 	foo.HypervisorConfig.NumVCPUs = 1
-	assert.False(deepCompare(foo, bar))
+	assert.False(utils.DeepCompare(foo, bar))
 	bar.HypervisorConfig.NumVCPUs = 1
-	assert.True(deepCompare(foo, bar))
+	assert.True(utils.DeepCompare(foo, bar))
 
 	// slice
 	foo.HypervisorConfig.KernelParams = []vc.Param{}
-	assert.True(deepCompare(foo, bar))
+	assert.True(utils.DeepCompare(foo, bar))
 	foo.HypervisorConfig.KernelParams = append(foo.HypervisorConfig.KernelParams, vc.Param{Key: "key", Value: "value"})
-	assert.False(deepCompare(foo, bar))
+	assert.False(utils.DeepCompare(foo, bar))
 	bar.HypervisorConfig.KernelParams = append(bar.HypervisorConfig.KernelParams, vc.Param{Key: "key", Value: "value"})
-	assert.True(deepCompare(foo, bar))
+	assert.True(utils.DeepCompare(foo, bar))
 
 	// map
 	var fooMap map[string]vc.VMConfig
 	var barMap map[string]vc.VMConfig
-	assert.False(deepCompare(foo, fooMap))
-	assert.True(deepCompare(fooMap, barMap))
+	assert.False(utils.DeepCompare(foo, fooMap))
+	assert.True(utils.DeepCompare(fooMap, barMap))
 	fooMap = make(map[string]vc.VMConfig)
-	assert.True(deepCompare(fooMap, barMap))
+	assert.True(utils.DeepCompare(fooMap, barMap))
 	fooMap["foo"] = foo
-	assert.False(deepCompare(fooMap, barMap))
+	assert.False(utils.DeepCompare(fooMap, barMap))
 	barMap = make(map[string]vc.VMConfig)
-	assert.False(deepCompare(fooMap, barMap))
+	assert.False(utils.DeepCompare(fooMap, barMap))
 	barMap["foo"] = bar
-	assert.True(deepCompare(fooMap, barMap))
+	assert.True(utils.DeepCompare(fooMap, barMap))
 
 	// invalid interface
 	var f1 vc.Factory
 	var f2 vc.Factory
 	var f3 base.FactoryBase
-	assert.True(deepCompare(f1, f2))
-	assert.True(deepCompare(f1, f3))
+	assert.True(utils.DeepCompare(f1, f2))
+	assert.True(utils.DeepCompare(f1, f3))
 
 	// valid interface
 	var config Config
@@ -331,8 +331,8 @@ func TestDeepCompare(t *testing.T) {
 	}
 	f1, err = NewFactory(ctx, config, false)
 	assert.Nil(err)
-	assert.True(deepCompare(f1, f1))
+	assert.True(utils.DeepCompare(f1, f1))
 	f2, err = NewFactory(ctx, config, false)
 	assert.Nil(err)
-	assert.False(deepCompare(f1, f2))
+	assert.False(utils.DeepCompare(f1, f2))
 }
diff --git a/virtcontainers/factory/grpcCache/grpcCache.go b/virtcontainers/factory/grpcCache/grpcCache.go
new file mode 100644
index 0000000000..e9bf37d8c0
--- /dev/null
+++ b/virtcontainers/factory/grpcCache/grpcCache.go
@@ -0,0 +1,62 @@
+// Copyright (c) 2019 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// grpcCache implements base vm factory that get base vm from grpc
+
+package grpcCache
+
+import (
+	"context"
+	"fmt"
+	google_protobuf "github.com/golang/protobuf/ptypes/empty"
+	pb "github.com/kata-containers/runtime/protocols/cache"
+	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/kata-containers/runtime/virtcontainers/factory/base"
+	"github.com/pkg/errors"
+	"google.golang.org/grpc"
+)
+
+type grpcCache struct {
+	conn   *grpc.ClientConn
+	config *vc.VMConfig
+}
+
+// New returns a new direct vm factory.
+func New(ctx context.Context, endpoint string) (base.FactoryBase, error) {
+	conn, err := grpc.Dial(fmt.Sprintf("unix://%s", endpoint), grpc.WithInsecure())
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to connect %q", endpoint)
+	}
+
+	jConfig, err := pb.NewCacheServiceClient(conn).Config(ctx, &google_protobuf.Empty{})
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to Config")
+	}
+
+	config, err := vc.GrpcToVMConfig(jConfig)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to convert JSON to VMConfig")
+	}
+
+	return &grpcCache{conn: conn, config: config}, nil
+}
+
+// Config returns the direct factory's configuration.
+func (g *grpcCache) Config() vc.VMConfig {
+	return *g.config
+}
+
+// GetBaseVM create a new VM directly.
+func (g *grpcCache) GetBaseVM(ctx context.Context, config vc.VMConfig) (*vc.VM, error) {
+	defer g.conn.Close()
+	gVM, err := pb.NewCacheServiceClient(g.conn).GetBaseVM(ctx, &google_protobuf.Empty{})
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to GetBaseVM")
+	}
+	return vc.NewVMFromGrpc(ctx, gVM, *g.config)
+}
+
+// CloseFactory closes the direct vm factory.
+func (g *grpcCache) CloseFactory(ctx context.Context) {
+}
diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go
index 9111fff653..cb401f0001 100644
--- a/virtcontainers/fc.go
+++ b/virtcontainers/fc.go
@@ -7,6 +7,7 @@ package virtcontainers
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/url"
 	"os/exec"
@@ -702,3 +703,11 @@ func (fc *firecracker) getThreadIDs() (*threadIDs, error) {
 func (fc *firecracker) cleanup() error {
 	return nil
 }
+
+func (fc *firecracker) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, storage resourceStorage, j []byte) error {
+	return errors.New("firecracker is not supported by VM cache")
+}
+
+func (fc *firecracker) toGrpc() ([]byte, error) {
+	return nil, errors.New("firecracker is not supported by VM cache")
+}
diff --git a/virtcontainers/hyperstart_agent.go b/virtcontainers/hyperstart_agent.go
index da6b9ecb69..dc9f3aed40 100644
--- a/virtcontainers/hyperstart_agent.go
+++ b/virtcontainers/hyperstart_agent.go
@@ -327,6 +327,10 @@ func (h *hyper) configure(hv hypervisor, id, sharePath string, builtin bool, con
 	return hv.addDevice(sharedVolume, fsDev)
 }
 
+func (h *hyper) configureFromGrpc(id string, builtin bool, config interface{}) error {
+	return nil
+}
+
 func (h *hyper) createSandbox(sandbox *Sandbox) (err error) {
 	return h.configure(sandbox.hypervisor, "", h.getSharePath(sandbox.id), false, nil)
 }
@@ -999,6 +1003,12 @@ func (h *hyper) setProxy(sandbox *Sandbox, proxy proxy, pid int, url string) err
 	return nil
 }
 
+func (h *hyper) setProxyFromGrpc(proxy proxy, pid int, url string) {
+	h.proxy = proxy
+	h.state.ProxyPid = pid
+	h.state.URL = url
+}
+
 func (h *hyper) getGuestDetails(*grpc.GuestDetailsRequest) (*grpc.GuestDetailsResponse, error) {
 	// hyperstart-agent does not support getGuestDetails
 	return nil, nil
diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go
index dd37f8706d..68c751ec9d 100644
--- a/virtcontainers/hypervisor.go
+++ b/virtcontainers/hypervisor.go
@@ -609,4 +609,6 @@ type hypervisor interface {
 	hypervisorConfig() HypervisorConfig
 	getThreadIDs() (*threadIDs, error)
 	cleanup() error
+	fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, storage resourceStorage, j []byte) error
+	toGrpc() ([]byte, error)
 }
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
index dcf4735d47..828e9e873c 100644
--- a/virtcontainers/kata_agent.go
+++ b/virtcontainers/kata_agent.go
@@ -221,7 +221,7 @@ func (k *kataAgent) capabilities() types.Capabilities {
 	return caps
 }
 
-func (k *kataAgent) configure(h hypervisor, id, sharePath string, builtin bool, config interface{}) error {
+func (k *kataAgent) internalConfigure(fromGrpc bool, h hypervisor, id, sharePath string, builtin bool, config interface{}) error {
 	if config != nil {
 		switch c := config.(type) {
 		case KataAgentConfig:
@@ -234,6 +234,14 @@ func (k *kataAgent) configure(h hypervisor, id, sharePath string, builtin bool,
 		}
 	}
 
+	if builtin {
+		k.proxyBuiltIn = true
+	}
+
+	if fromGrpc {
+		return nil
+	}
+
 	switch s := k.vmSocket.(type) {
 	case types.Socket:
 		err := h.addDevice(s, serialPortDev)
@@ -255,10 +263,6 @@ func (k *kataAgent) configure(h hypervisor, id, sharePath string, builtin bool,
 		return fmt.Errorf("Invalid config type")
 	}
 
-	if builtin {
-		k.proxyBuiltIn = true
-	}
-
 	// Neither create shared directory nor add 9p device if hypervisor
 	// doesn't support filesystem sharing.
 	caps := h.capabilities()
@@ -280,6 +284,14 @@ func (k *kataAgent) configure(h hypervisor, id, sharePath string, builtin bool,
 	return h.addDevice(sharedVolume, fsDev)
 }
 
+func (k *kataAgent) configure(h hypervisor, id, sharePath string, builtin bool, config interface{}) error {
+	return k.internalConfigure(false, h, id, sharePath, builtin, config)
+}
+
+func (k *kataAgent) configureFromGrpc(id string, builtin bool, config interface{}) error {
+	return k.internalConfigure(true, nil, id, "", builtin, config)
+}
+
 func (k *kataAgent) createSandbox(sandbox *Sandbox) error {
 	span, _ := k.trace("createSandbox")
 	defer span.Finish()
@@ -587,6 +599,12 @@ func (k *kataAgent) setProxy(sandbox *Sandbox, proxy proxy, pid int, url string)
 	return nil
 }
 
+func (k *kataAgent) setProxyFromGrpc(proxy proxy, pid int, url string) {
+	k.proxy = proxy
+	k.state.ProxyPid = pid
+	k.state.URL = url
+}
+
 func (k *kataAgent) startSandbox(sandbox *Sandbox) error {
 	span, _ := k.trace("startSandbox")
 	defer span.Finish()
diff --git a/virtcontainers/kata_agent.go.rej b/virtcontainers/kata_agent.go.rej
new file mode 100644
index 0000000000..7bafe79a25
--- /dev/null
+++ b/virtcontainers/kata_agent.go.rej
@@ -0,0 +1,116 @@
+***************
+*** 234,283 ****
+  		}
+  	}
+  
+- 	switch s := k.vmSocket.(type) {
+- 	case types.Socket:
+- 		err := h.addDevice(s, serialPortDev)
+- 		if err != nil {
+- 			return err
+  		}
+- 	case kataVSOCK:
+- 		var err error
+- 		s.vhostFd, s.contextID, err = utils.FindContextID()
+- 		if err != nil {
+- 			return err
+  		}
+- 		s.port = uint32(vSockPort)
+- 		if err := h.addDevice(s, vSockPCIDev); err != nil {
+- 			return err
+  		}
+- 		k.vmSocket = s
+- 	default:
+- 		return fmt.Errorf("Invalid config type")
+- 	}
+  
+- 	if builtin {
+- 		k.proxyBuiltIn = true
+- 	}
+  
+- 	// Neither create shared directory nor add 9p device if hypervisor
+- 	// doesn't support filesystem sharing.
+- 	caps := h.capabilities()
+- 	if !caps.IsFsSharingSupported() {
+- 		return nil
+  	}
+  
+- 	// Create shared directory and add the shared volume if filesystem sharing is supported.
+- 	// This volume contains all bind mounted container bundles.
+- 	sharedVolume := types.Volume{
+- 		MountTag: mountGuest9pTag,
+- 		HostPath: sharePath,
+- 	}
+  
+- 	if err := os.MkdirAll(sharedVolume.HostPath, dirMode); err != nil {
+- 		return err
+- 	}
+  
+- 	return h.addDevice(sharedVolume, fsDev)
+  }
+  
+  func (k *kataAgent) createSandbox(sandbox *Sandbox) error {
+--- 234,296 ----
+  		}
+  	}
+  
++ 	if builtin {
++ 		k.proxyBuiltIn = true
++ 	}
++ 
++ 	var err error
++ 	if !fromGrpc {
++ 		switch s := k.vmSocket.(type) {
++ 		case types.Socket:
++ 			err = h.addDevice(s, serialPortDev)
++ 			if err != nil {
++ 				return err
++ 			}
++ 		case kataVSOCK:
++ 			var err error
++ 			s.vhostFd, s.contextID, err = utils.FindContextID()
++ 			if err != nil {
++ 				return err
++ 			}
++ 			s.port = uint32(vSockPort)
++ 			if err = h.addDevice(s, vSockPCIDev); err != nil {
++ 				return err
++ 			}
++ 			k.vmSocket = s
++ 		default:
++ 			return fmt.Errorf("Invalid config type")
+  		}
++ 
++ 		// Neither create shared directory nor add 9p device if hypervisor
++ 		// doesn't support filesystem sharing.
++ 		caps := h.capabilities()
++ 		if !caps.IsFsSharingSupported() {
++ 			return nil
+  		}
++ 
++ 		// Create shared directory and add the shared volume if filesystem sharing is supported.
++ 		// This volume contains all bind mounted container bundles.
++ 		sharedVolume := types.Volume{
++ 			MountTag: mountGuest9pTag,
++ 			HostPath: sharePath,
+  		}
+  
++ 		if err = os.MkdirAll(sharedVolume.HostPath, dirMode); err != nil {
++ 			return err
++ 		}
+  
++ 		err = h.addDevice(sharedVolume, fsDev)
+  	}
+  
++ 	return err
++ }
+  
++ func (k *kataAgent) configure(h hypervisor, id, sharePath string, builtin bool, config interface{}) error {
++ 	return k.internalConfigure(false, h, id, sharePath, builtin, config)
++ }
+  
++ func (k *kataAgent) configureFromGrpc(id string, builtin bool, config interface{}) error {
++ 	return k.internalConfigure(true, nil, id, "", builtin, config)
+  }
+  
+  func (k *kataAgent) createSandbox(sandbox *Sandbox) error {
diff --git a/virtcontainers/mock_hypervisor.go b/virtcontainers/mock_hypervisor.go
index 37dccaf901..ad8bf0c6ea 100644
--- a/virtcontainers/mock_hypervisor.go
+++ b/virtcontainers/mock_hypervisor.go
@@ -7,6 +7,7 @@ package virtcontainers
 
 import (
 	"context"
+	"errors"
 	"os"
 
 	"github.com/kata-containers/runtime/virtcontainers/store"
@@ -100,3 +101,11 @@ func (m *mockHypervisor) getThreadIDs() (*threadIDs, error) {
 func (m *mockHypervisor) cleanup() error {
 	return nil
 }
+
+func (m *mockHypervisor) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, storage resourceStorage, j []byte) error {
+	return errors.New("mockHypervisor is not supported by VM cache")
+}
+
+func (m *mockHypervisor) toGrpc() ([]byte, error) {
+	return nil, errors.New("firecracker is not supported by VM cache")
+}
diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go
index 6ba15d4b33..db9a3dcbdf 100644
--- a/virtcontainers/noop_agent.go
+++ b/virtcontainers/noop_agent.go
@@ -171,6 +171,10 @@ func (n *noopAgent) configure(h hypervisor, id, sharePath string, builtin bool,
 	return nil
 }
 
+func (n *noopAgent) configureFromGrpc(id string, builtin bool, config interface{}) error {
+	return nil
+}
+
 // getVMPath is the Noop agent vm path getter. It does nothing.
 func (n *noopAgent) getVMPath(id string) string {
 	return ""
@@ -201,6 +205,9 @@ func (n *noopAgent) setProxy(sandbox *Sandbox, proxy proxy, pid int, url string)
 	return nil
 }
 
+func (n *noopAgent) setProxyFromGrpc(proxy proxy, pid int, url string) {
+}
+
 // getGuestDetails is the Noop agent GuestDetails queryer. It does nothing.
 func (n *noopAgent) getGuestDetails(*grpc.GuestDetailsRequest) (*grpc.GuestDetailsResponse, error) {
 	return nil, nil
diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go
index 4ffb4ec957..73ea4759bf 100644
--- a/virtcontainers/pkg/oci/utils.go
+++ b/virtcontainers/pkg/oci/utils.go
@@ -96,6 +96,12 @@ type CompatOCISpec struct {
 type FactoryConfig struct {
 	// Template enables VM templating support in VM factory.
 	Template bool
+
+	// VMCache enables VM cache support in VM factory.
+	VMCache bool
+
+	// VMCacheEndpoint specifies the endpoint of transport VM from the VM cache server to runtime.
+	VMCacheEndpoint string
 }
 
 // RuntimeConfig aggregates all runtime specific settings
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
index 2220108385..a515f56ad1 100644
--- a/virtcontainers/qemu.go
+++ b/virtcontainers/qemu.go
@@ -7,6 +7,7 @@ package virtcontainers
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"math"
 	"os"
@@ -1567,3 +1568,55 @@ func (q *qemu) cleanup() error {
 
 	return nil
 }
+
+type qemuGrpc struct {
+	ID             string
+	QmpChannelpath string
+	State          QemuState
+	NvdimmCount    int
+
+	// Most members of q.qemuConfig are just to generate
+	// q.qemuConfig.qemuParams that is used by LaunchQemu except
+	// q.qemuConfig.SMP.
+	// So just transport q.qemuConfig.SMP from VM Cache server to runtime.
+	QemuSMP govmmQemu.SMP
+}
+
+func (q *qemu) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, storage resourceStorage, j []byte) error {
+	var qp qemuGrpc
+	err := json.Unmarshal(j, &qp)
+	if err != nil {
+		return err
+	}
+
+	q.id = qp.ID
+	q.storage = storage
+	q.config = *hypervisorConfig
+	q.qmpMonitorCh.ctx = ctx
+	q.qmpMonitorCh.path = qp.QmpChannelpath
+	q.qemuConfig.Ctx = ctx
+	q.state = qp.State
+	q.arch = newQemuArch(q.config)
+	q.ctx = ctx
+	q.nvdimmCount = qp.NvdimmCount
+
+	q.qemuConfig.SMP = qp.QemuSMP
+
+	return nil
+}
+
+func (q *qemu) toGrpc() ([]byte, error) {
+	q.qmpShutdown()
+
+	q.cleanup()
+	qp := qemuGrpc{
+		ID:             q.id,
+		QmpChannelpath: q.qmpMonitorCh.path,
+		State:          q.state,
+		NvdimmCount:    q.nvdimmCount,
+
+		QemuSMP: q.qemuConfig.SMP,
+	}
+
+	return json.Marshal(&qp)
+}
diff --git a/virtcontainers/qemu_test.go b/virtcontainers/qemu_test.go
index 6c79a9297a..067a9dfca9 100644
--- a/virtcontainers/qemu_test.go
+++ b/virtcontainers/qemu_test.go
@@ -424,3 +424,22 @@ func TestQemuCleanup(t *testing.T) {
 	err := q.cleanup()
 	assert.Nil(err)
 }
+
+func TestQemuGrpc(t *testing.T) {
+	assert := assert.New(t)
+
+	config := newQemuConfig()
+	q := &qemu{
+		id:     "testqemu",
+		config: config,
+	}
+
+	json, err := q.toGrpc()
+	assert.Nil(err)
+
+	var q2 qemu
+	err = q2.fromGrpc(context.Background(), &config, nil, json)
+	assert.Nil(err)
+
+	assert.True(q.id == q2.id)
+}
diff --git a/virtcontainers/utils/compare.go b/virtcontainers/utils/compare.go
new file mode 100644
index 0000000000..e67b6bc476
--- /dev/null
+++ b/virtcontainers/utils/compare.go
@@ -0,0 +1,74 @@
+// Copyright (c) 2019 HyperHQ Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package utils
+
+import "reflect"
+
+func compareStruct(foo, bar reflect.Value) bool {
+	for i := 0; i < foo.NumField(); i++ {
+		if !deepCompareValue(foo.Field(i), bar.Field(i)) {
+			return false
+		}
+	}
+
+	return true
+}
+
+func compareMap(foo, bar reflect.Value) bool {
+	if foo.Len() != bar.Len() {
+		return false
+	}
+
+	for _, k := range foo.MapKeys() {
+		if !deepCompareValue(foo.MapIndex(k), bar.MapIndex(k)) {
+			return false
+		}
+	}
+
+	return true
+}
+
+func compareSlice(foo, bar reflect.Value) bool {
+	if foo.Len() != bar.Len() {
+		return false
+	}
+	for j := 0; j < foo.Len(); j++ {
+		if !deepCompareValue(foo.Index(j), bar.Index(j)) {
+			return false
+		}
+	}
+	return true
+}
+
+func deepCompareValue(foo, bar reflect.Value) bool {
+	if !foo.IsValid() || !bar.IsValid() {
+		return foo.IsValid() == bar.IsValid()
+	}
+
+	if foo.Type() != bar.Type() {
+		return false
+	}
+	switch foo.Kind() {
+	case reflect.Map:
+		return compareMap(foo, bar)
+	case reflect.Array:
+		fallthrough
+	case reflect.Slice:
+		return compareSlice(foo, bar)
+	case reflect.Struct:
+		return compareStruct(foo, bar)
+	default:
+		return foo.Interface() == bar.Interface()
+	}
+}
+
+// DeepCompare compare foo and bar.
+func DeepCompare(foo, bar interface{}) bool {
+	v1 := reflect.ValueOf(foo)
+	v2 := reflect.ValueOf(bar)
+
+	return deepCompareValue(v1, v2)
+}
diff --git a/virtcontainers/vm.go b/virtcontainers/vm.go
index e04f521727..67451a8038 100644
--- a/virtcontainers/vm.go
+++ b/virtcontainers/vm.go
@@ -7,10 +7,13 @@ package virtcontainers
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
 	"os"
 	"path/filepath"
 	"time"
 
+	pb "github.com/kata-containers/runtime/protocols/cache"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
 	"github.com/kata-containers/runtime/virtcontainers/store"
 	"github.com/sirupsen/logrus"
@@ -52,6 +55,63 @@ func (c *VMConfig) Valid() error {
 	return c.HypervisorConfig.valid()
 }
 
+// ToGrpc convert VMConfig struct to grpc format pb.GrpcVMConfig.
+func (c *VMConfig) ToGrpc() (*pb.GrpcVMConfig, error) {
+	data, err := json.Marshal(&c)
+	if err != nil {
+		return nil, err
+	}
+
+	var agentConfig []byte
+	switch aconf := c.AgentConfig.(type) {
+	case HyperConfig:
+		agentConfig, err = json.Marshal(&aconf)
+	case KataAgentConfig:
+		agentConfig, err = json.Marshal(&aconf)
+	default:
+		err = fmt.Errorf("agent type %s is not supported by VM cache", c.AgentType)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return &pb.GrpcVMConfig{
+		Data:        data,
+		AgentConfig: agentConfig,
+	}, nil
+}
+
+// GrpcToVMConfig convert grpc format pb.GrpcVMConfig to VMConfig struct.
+func GrpcToVMConfig(j *pb.GrpcVMConfig) (*VMConfig, error) {
+	var config VMConfig
+	err := json.Unmarshal(j.Data, &config)
+	if err != nil {
+		return nil, err
+	}
+
+	switch config.AgentType {
+	case HyperstartAgent:
+		var hyperConfig HyperConfig
+		err := json.Unmarshal(j.AgentConfig, &hyperConfig)
+		if err == nil {
+			config.AgentConfig = hyperConfig
+		}
+	case KataContainersAgent:
+		var kataConfig KataAgentConfig
+		err := json.Unmarshal(j.AgentConfig, &kataConfig)
+		if err == nil {
+			config.AgentConfig = kataConfig
+		}
+	default:
+		err = fmt.Errorf("agent type %s is not supported by VM cache", config.AgentType)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return &config, nil
+}
+
 func setupProxy(h hypervisor, agent agent, config VMConfig, id string) (int, string, proxy, error) {
 	consoleURL, err := h.getSandboxConsole(id)
 	if err != nil {
@@ -192,6 +252,41 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) {
 	}, nil
 }
 
+// NewVMFromGrpc creates a new VM based on provided pb.GrpcVM and VMConfig.
+func NewVMFromGrpc(ctx context.Context, v *pb.GrpcVM, config VMConfig) (*VM, error) {
+	virtLog.WithField("GrpcVM", v).WithField("config", config).Info("create new vm from Grpc")
+
+	hypervisor, err := newHypervisor(config.HypervisorType)
+	if err != nil {
+		return nil, err
+	}
+	err = hypervisor.fromGrpc(ctx, &config.HypervisorConfig, &filesystem{}, v.Hypervisor)
+	if err != nil {
+		return nil, err
+	}
+
+	agent := newAgent(config.AgentType)
+	agent.configureFromGrpc(v.Id, isProxyBuiltIn(config.ProxyType), config.AgentConfig)
+
+	proxy, err := newProxy(config.ProxyType)
+	if err != nil {
+		return nil, err
+	}
+	agent.setProxyFromGrpc(proxy, int(v.ProxyPid), v.ProxyURL)
+
+	return &VM{
+		id:         v.Id,
+		hypervisor: hypervisor,
+		agent:      agent,
+		proxy:      proxy,
+		proxyPid:   int(v.ProxyPid),
+		proxyURL:   v.ProxyURL,
+		cpu:        v.Cpu,
+		memory:     v.Memory,
+		cpuDelta:   v.CpuDelta,
+	}, nil
+}
+
 func buildVMSharePath(id string) string {
 	return filepath.Join(store.RunVMStoragePath, id, "shared")
 }
@@ -358,3 +453,23 @@ func (v *VM) assignSandbox(s *Sandbox) error {
 
 	return nil
 }
+
+// ToGrpc convert VM struct to Grpc format pb.GrpcVM.
+func (v *VM) ToGrpc(config VMConfig) (*pb.GrpcVM, error) {
+	hJSON, err := v.hypervisor.toGrpc()
+	if err != nil {
+		return nil, err
+	}
+
+	return &pb.GrpcVM{
+		Id:         v.id,
+		Hypervisor: hJSON,
+
+		ProxyPid: int64(v.proxyPid),
+		ProxyURL: v.proxyURL,
+
+		Cpu:      v.cpu,
+		Memory:   v.memory,
+		CpuDelta: v.cpuDelta,
+	}, nil
+}
diff --git a/virtcontainers/vm_test.go b/virtcontainers/vm_test.go
index 40a9729d6b..5e6117893a 100644
--- a/virtcontainers/vm_test.go
+++ b/virtcontainers/vm_test.go
@@ -10,6 +10,7 @@ import (
 	"io/ioutil"
 	"testing"
 
+	"github.com/kata-containers/runtime/virtcontainers/utils"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -110,3 +111,22 @@ func TestSetupProxy(t *testing.T) {
 	_, _, _, err = setupProxy(hypervisor, agent, config, "foobar")
 	assert.Nil(err)
 }
+
+func TestVMConfigGrpc(t *testing.T) {
+	assert := assert.New(t)
+	config := VMConfig{
+		HypervisorType:   QemuHypervisor,
+		HypervisorConfig: newQemuConfig(),
+		AgentType:        KataContainersAgent,
+		AgentConfig:      KataAgentConfig{false, true},
+		ProxyType:        NoopProxyType,
+	}
+
+	p, err := config.ToGrpc()
+	assert.Nil(err)
+
+	config2, err := GrpcToVMConfig(p)
+	assert.Nil(err)
+
+	assert.True(utils.DeepCompare(config, *config2))
+}