From c60b536a01c4b92734791e439b2a3bdf20cd235a Mon Sep 17 00:00:00 2001 From: Frank Yang Date: Mon, 19 Mar 2018 15:46:43 +0800 Subject: [PATCH] feature: add plugin framework which supports executing custom code at plugin points like start and stop in life cyle of container and daemon --- apis/plugins/ContainerPlugin.go | 14 ++++ apis/plugins/DaemonPlugin.go | 12 +++ apis/server/container_bridge.go | 12 ++- apis/server/server.go | 16 ++-- daemon/config/config.go | 3 + daemon/daemon.go | 105 +++++++++++++++++++++---- daemon/mgr/container.go | 47 +++++++---- daemon/mgr/spec.go | 8 +- daemon/mgr/spec_hook.go | 57 +++++++++++++- docs/features/pouch_with_plugin.md | 121 +++++++++++++++++++++++++++++ internal/generator.go | 4 +- main.go | 3 +- 12 files changed, 355 insertions(+), 47 deletions(-) create mode 100644 apis/plugins/ContainerPlugin.go create mode 100644 apis/plugins/DaemonPlugin.go create mode 100644 docs/features/pouch_with_plugin.md diff --git a/apis/plugins/ContainerPlugin.go b/apis/plugins/ContainerPlugin.go new file mode 100644 index 000000000..6f4cf801a --- /dev/null +++ b/apis/plugins/ContainerPlugin.go @@ -0,0 +1,14 @@ +package plugins + +import "io" + +// ContainerPlugin defines in which place a plugin will be triggered in container lifecycle +type ContainerPlugin interface { + // PreCreate defines plugin point where recevives an container create request, in this plugin point user + // could change the container create body passed-in by http request body + PreCreate(io.ReadCloser) (io.ReadCloser, error) + + // PreStart returns an array of priority and args which will pass to runc, the every priority + // used to sort the pre start array that pass to runc, network plugin hook always has priority value 0. + PreStart(interface{}) ([]int, [][]string, error) +} diff --git a/apis/plugins/DaemonPlugin.go b/apis/plugins/DaemonPlugin.go new file mode 100644 index 000000000..42581035c --- /dev/null +++ b/apis/plugins/DaemonPlugin.go @@ -0,0 +1,12 @@ +package plugins + +// DaemonPlugin defines in which place does pouch daemon support plugin +type DaemonPlugin interface { + // PreStartHook is invoked by pouch daemon before real start, in this hook user could start dfget proxy or other + // standalone process plugins + PreStartHook() error + + // PreStopHook is invoked by pouch daemon before daemon process exit, not a promise if daemon is killed, in this + // hook user could stop the process or plugin started by PreStartHook + PreStopHook() error +} diff --git a/apis/server/container_bridge.go b/apis/server/container_bridge.go index 3fc9d6413..36e6dbcfd 100644 --- a/apis/server/container_bridge.go +++ b/apis/server/container_bridge.go @@ -16,6 +16,8 @@ import ( "github.com/go-openapi/strfmt" "github.com/gorilla/mux" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) func (s *Server) removeContainers(ctx context.Context, rw http.ResponseWriter, req *http.Request) error { @@ -114,8 +116,16 @@ func (s *Server) startContainerExec(ctx context.Context, rw http.ResponseWriter, func (s *Server) createContainer(ctx context.Context, rw http.ResponseWriter, req *http.Request) error { config := &types.ContainerCreateConfig{} + reader := req.Body + var ex error + if s.ContainerPlugin != nil { + logrus.Infof("invoke container pre-create hook in plugin") + if reader, ex = s.ContainerPlugin.PreCreate(req.Body); ex != nil { + return errors.Wrapf(ex, "pre-create plugin piont execute failed") + } + } // decode request body - if err := json.NewDecoder(req.Body).Decode(config); err != nil { + if err := json.NewDecoder(reader).Decode(config); err != nil { return httputils.NewHTTPError(err, http.StatusBadRequest) } // validate request body diff --git a/apis/server/server.go b/apis/server/server.go index a8e7a65e3..b0cef954e 100644 --- a/apis/server/server.go +++ b/apis/server/server.go @@ -9,6 +9,7 @@ import ( "strings" "syscall" + "github.com/alibaba/pouch/apis/plugins" "github.com/alibaba/pouch/daemon/config" "github.com/alibaba/pouch/daemon/mgr" @@ -18,13 +19,14 @@ import ( // Server is a http server which serves restful api to client. type Server struct { - Config config.Config - ContainerMgr mgr.ContainerMgr - SystemMgr mgr.SystemMgr - ImageMgr mgr.ImageMgr - VolumeMgr mgr.VolumeMgr - NetworkMgr mgr.NetworkMgr - listeners []net.Listener + Config config.Config + ContainerMgr mgr.ContainerMgr + SystemMgr mgr.SystemMgr + ImageMgr mgr.ImageMgr + VolumeMgr mgr.VolumeMgr + NetworkMgr mgr.NetworkMgr + listeners []net.Listener + ContainerPlugin plugins.ContainerPlugin } // Start setup route table and listen to specified address which currently only supports unix socket and tcp address. diff --git a/daemon/config/config.go b/daemon/config/config.go index 590fc893e..bf185617c 100644 --- a/daemon/config/config.go +++ b/daemon/config/config.go @@ -66,4 +66,7 @@ type Config struct { // CgroupParent is to set parent cgroup for all containers CgroupParent string `json:"cgroup-parent,omitempty"` + + // PluginPath is set the path where plugin so file put + PluginPath string `json:"plugin"` } diff --git a/daemon/daemon.go b/daemon/daemon.go index 6c89be93b..e0e690adc 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -2,9 +2,12 @@ package daemon import ( "context" + "fmt" "path" + "plugin" "reflect" + "github.com/alibaba/pouch/apis/plugins" "github.com/alibaba/pouch/apis/server" cri "github.com/alibaba/pouch/cri/service" "github.com/alibaba/pouch/ctrd" @@ -15,22 +18,25 @@ import ( "github.com/alibaba/pouch/pkg/meta" "github.com/gorilla/mux" + "github.com/pkg/errors" "github.com/sirupsen/logrus" ) // Daemon refers to a daemon. type Daemon struct { - config config.Config - containerStore *meta.Store - containerd *ctrd.Client - containerMgr mgr.ContainerMgr - systemMgr mgr.SystemMgr - imageMgr mgr.ImageMgr - volumeMgr mgr.VolumeMgr - networkMgr mgr.NetworkMgr - criMgr mgr.CriMgr - server server.Server - criService *cri.Service + config config.Config + containerStore *meta.Store + containerd *ctrd.Client + containerMgr mgr.ContainerMgr + systemMgr mgr.SystemMgr + imageMgr mgr.ImageMgr + volumeMgr mgr.VolumeMgr + networkMgr mgr.NetworkMgr + criMgr mgr.CriMgr + server server.Server + criService *cri.Service + containerPlugin plugins.ContainerPlugin + daemonPlugin plugins.DaemonPlugin } // router represents the router of daemon. @@ -71,11 +77,58 @@ func NewDaemon(cfg config.Config) *Daemon { } } +func loadSymbolByName(p *plugin.Plugin, name string) (plugin.Symbol, error) { + s, err := p.Lookup(name) + if err != nil { + return nil, errors.Wrapf(err, "lookup plugin with name %s error", name) + } + return s, nil +} + // Run starts daemon. func (d *Daemon) Run() error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() + var s plugin.Symbol + var err error + + if d.config.PluginPath != "" { + p, err := plugin.Open(d.config.PluginPath) + if err != nil { + return errors.Wrapf(err, "load plugin at %s error", d.config.PluginPath) + } + + //load container plugin if exist + if s, err = loadSymbolByName(p, "DaemonPlugin"); err != nil { + return err + } + if daemonPlugin, ok := s.(plugins.DaemonPlugin); ok { + logrus.Infof("setup daemon plugin from %s", d.config.PluginPath) + d.daemonPlugin = daemonPlugin + } else if s != nil { + return fmt.Errorf("not a container plugin at %s %q", d.config.PluginPath, s) + } + + //load container plugin if exist + if s, err = loadSymbolByName(p, "ContainerPlugin"); err != nil { + return err + } + if containerPlugin, ok := s.(plugins.ContainerPlugin); ok { + logrus.Infof("setup container plugin from %s", d.config.PluginPath) + d.containerPlugin = containerPlugin + } else if s != nil { + return fmt.Errorf("not a container plugin at %s %q", d.config.PluginPath, s) + } + } + + if d.daemonPlugin != nil { + logrus.Infof("invoke pre-start hook in plugin") + if err = d.daemonPlugin.PreStartHook(); err != nil { + return err + } + } + imageMgr, err := internal.GenImageMgr(&d.config, d) if err != nil { return err @@ -118,12 +171,13 @@ func (d *Daemon) Run() error { } d.server = server.Server{ - Config: d.config, - ContainerMgr: containerMgr, - SystemMgr: systemMgr, - ImageMgr: imageMgr, - VolumeMgr: volumeMgr, - NetworkMgr: networkMgr, + Config: d.config, + ContainerMgr: containerMgr, + SystemMgr: systemMgr, + ImageMgr: imageMgr, + VolumeMgr: volumeMgr, + NetworkMgr: networkMgr, + ContainerPlugin: d.containerPlugin, } // init base network @@ -166,6 +220,7 @@ func (d *Daemon) Run() error { logrus.Infof("GRPC server stopped") <-streamServerCloseCh logrus.Infof("Stream server stopped") + return nil } @@ -212,3 +267,19 @@ func (d *Daemon) MetaStore() *meta.Store { func (d *Daemon) networkInit(ctx context.Context) error { return mode.NetworkModeInit(ctx, d.config.NetworkConfg, d.networkMgr) } + +// ContainerPlugin returns the container plugin fetched from shared file +func (d *Daemon) ContainerPlugin() plugins.ContainerPlugin { + return d.containerPlugin +} + +// ShutdownPlugin invoke pre-stop method in daemon plugin if exist +func (d *Daemon) ShutdownPlugin() error { + if d.daemonPlugin != nil { + logrus.Infof("invoke pre-stop hook in plugin") + if err := d.daemonPlugin.PreStopHook(); err != nil { + logrus.Errorf("stop prehook execute error %v", err) + } + } + return nil +} diff --git a/daemon/mgr/container.go b/daemon/mgr/container.go index 9533f063c..6b4f6689a 100644 --- a/daemon/mgr/container.go +++ b/daemon/mgr/container.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/alibaba/pouch/apis/plugins" "github.com/alibaba/pouch/apis/types" "github.com/alibaba/pouch/ctrd" "github.com/alibaba/pouch/daemon/config" @@ -112,22 +113,25 @@ type ContainerManager struct { // monitor is used to handle container's event, eg: exit, stop and so on. monitor *ContainerMonitor + + containerPlugin plugins.ContainerPlugin } // NewContainerManager creates a brand new container manager. -func NewContainerManager(ctx context.Context, store *meta.Store, cli *ctrd.Client, imgMgr ImageMgr, volMgr VolumeMgr, netMgr NetworkMgr, cfg *config.Config) (*ContainerManager, error) { +func NewContainerManager(ctx context.Context, store *meta.Store, cli *ctrd.Client, imgMgr ImageMgr, volMgr VolumeMgr, netMgr NetworkMgr, cfg *config.Config, contPlugin plugins.ContainerPlugin) (*ContainerManager, error) { mgr := &ContainerManager{ - Store: store, - NameToID: collect.NewSafeMap(), - Client: cli, - ImageMgr: imgMgr, - VolumeMgr: volMgr, - NetworkMgr: netMgr, - IOs: containerio.NewCache(), - ExecProcesses: collect.NewSafeMap(), - cache: collect.NewSafeMap(), - Config: cfg, - monitor: NewContainerMonitor(), + Store: store, + NameToID: collect.NewSafeMap(), + Client: cli, + ImageMgr: imgMgr, + VolumeMgr: volMgr, + NetworkMgr: netMgr, + IOs: containerio.NewCache(), + ExecProcesses: collect.NewSafeMap(), + cache: collect.NewSafeMap(), + Config: cfg, + monitor: NewContainerMonitor(), + containerPlugin: contPlugin, } mgr.Client.SetExitHooks(mgr.exitedAndRelease) @@ -506,11 +510,22 @@ func (mgr *ContainerManager) Start(ctx context.Context, id, detachKeys string) ( s.Linux.CgroupsPath = filepath.Join(cgroupsParent, c.ID()) } + var prioArr []int + var argsArr [][]string + if mgr.containerPlugin != nil { + prioArr, argsArr, err = mgr.containerPlugin.PreStart(c) + if err != nil { + return errors.Wrapf(err, "get pre-start hook error from container plugin") + } + } + sw := &SpecWrapper{ - s: s, - ctrMgr: mgr, - volMgr: mgr.VolumeMgr, - netMgr: mgr.NetworkMgr, + s: s, + ctrMgr: mgr, + volMgr: mgr.VolumeMgr, + netMgr: mgr.NetworkMgr, + prioArr: prioArr, + argsArr: argsArr, } for _, setup := range SetupFuncs() { diff --git a/daemon/mgr/spec.go b/daemon/mgr/spec.go index cdb2dc099..a98acbed3 100644 --- a/daemon/mgr/spec.go +++ b/daemon/mgr/spec.go @@ -10,9 +10,11 @@ import ( type SpecWrapper struct { s *specs.Spec - ctrMgr ContainerMgr - volMgr VolumeMgr - netMgr NetworkMgr + ctrMgr ContainerMgr + volMgr VolumeMgr + netMgr NetworkMgr + prioArr []int + argsArr [][]string } // SetupFunc defines spec setup function type. diff --git a/daemon/mgr/spec_hook.go b/daemon/mgr/spec_hook.go index b5cd2a692..4bcb52c10 100644 --- a/daemon/mgr/spec_hook.go +++ b/daemon/mgr/spec_hook.go @@ -2,13 +2,29 @@ package mgr import ( "context" + "sort" "strings" specs "github.com/opencontainers/runtime-spec/specs-go" ) -//if set rich mode, set initscript +//setup hooks specified by user via plugins, if set rich mode and init-script exists set init-script func setupHook(ctx context.Context, c *ContainerMeta, spec *SpecWrapper) error { + if len(spec.argsArr) > 0 { + var hookArr []*wrapperEmbedPrestart + for i, hook := range spec.s.Hooks.Prestart { + hookArr = append(hookArr, &wrapperEmbedPrestart{-i, append([]string{hook.Path}, hook.Args...)}) + } + priorityArr := spec.prioArr + argsArr := spec.argsArr + for i, p := range priorityArr { + hookArr = append(hookArr, &wrapperEmbedPrestart{p, argsArr[i]}) + } + sortedArr := hookArray(hookArr) + sort.Sort(sortedArr) + spec.s.Hooks.Prestart = sortedArr.toOciPrestartHook() + } + if !c.Config.Rich || c.Config.InitScript == "" { return nil } @@ -35,3 +51,42 @@ func setupHook(ctx context.Context, c *ContainerMeta, spec *SpecWrapper) error { return nil } + +type hookArray []*wrapperEmbedPrestart + +// Len is defined in order to support sort +func (h hookArray) Len() int { + return len(h) +} + +// Len is defined in order to support sort +func (h hookArray) Swap(i, j int) { + h[i], h[j] = h[j], h[i] +} + +// Less is defined in order to support sort, bigger priority execute first +func (h hookArray) Less(i, j int) bool { + return h[i].Priority()-h[j].Priority() > 0 +} + +func (h hookArray) toOciPrestartHook() []specs.Hook { + allHooks := make([]specs.Hook, len(h)) + for i, hook := range h { + allHooks[i].Path = hook.Hook()[0] + allHooks[i].Args = hook.Hook()[1:] + } + return allHooks +} + +type wrapperEmbedPrestart struct { + p int + args []string +} + +func (w *wrapperEmbedPrestart) Priority() int { + return w.p +} + +func (w *wrapperEmbedPrestart) Hook() []string { + return w.args +} diff --git a/docs/features/pouch_with_plugin.md b/docs/features/pouch_with_plugin.md new file mode 100644 index 000000000..e32f7a84d --- /dev/null +++ b/docs/features/pouch_with_plugin.md @@ -0,0 +1,121 @@ +# Pouch with plugin +In order to run custom code at some point, we support a plugin frame work which introduced from golang 1.8. At this time in this plugin frame work we enable user to add custom code at four point: +* pre-start daemon point +* pre-stop daemon point +* pre-create container point +* pre-start container point + +Above four point orgnized by two Plugin, which are DaemonPlugin and ContainerPlugin: +``` +// DaemonPlugin defines in which place does pouch daemon support plugin +type DaemonPlugin interface { + // PreStartHook is invoked by pouch daemon before real start, in this hook user could start dfget proxy or other + // standalone process plugins + PreStartHook() error + + // PreStopHook is invoked by pouch daemon before daemon process exit, not a promise if daemon is killed, in this + // hook user could stop the process or plugin started by PreStartHook + PreStopHook() error +} + +// ContainerPlugin defines in which place a plugin will be triggered in container lifecycle +type ContainerPlugin interface { + // PreCreate defines plugin point where recevives an container create request, in this plugin point user + // could change the container create body passed-in by http request body + PreCreate(io.ReadCloser) (io.ReadCloser, error) + + // PreStart returns an array of priority and args which will pass to runc, the every priority + // used to sort the pre start array that pass to runc, network plugin hook always has priority value 0. + PreStart(interface{}) ([]int, [][]string, error) +} + +``` +These two Plugin symbol will be fetch by name `DaemonPlugin` and `ContainerPlugin` like this: +``` +daemonPlugin, _ := p.Lookup("DaemonPlugin") +containerPlugin, _ := p.Lookup("ContainerPlugin") +``` + +## example +define two plugin only print some log at correspond point +``` +package main + +import ( + "fmt" + "io" +) + +var ContainerPlugin ContPlugin + +type ContPlugin int + +var DaemonPlugin DPlugin + +type DPlugin int + +func (d DPlugin) PreStartHook() error { + fmt.Println("pre-start hook in daemon is called") + return nil +} + +func (d DPlugin) PreStopHook() error { + fmt.Println("pre-stop hook in daemon is called") + return nil +} + +func (c ContPlugin) PreCreate(in io.ReadCloser) (io.ReadCloser, error) { + fmt.Println("pre create method called") + return in, nil +} + +func (c ContPlugin) PreStart(interface{}) ([]int, [][]string, error) { + fmt.Println("pre start method called") + return []int{-4}, [][]string{{"/usr/bin/touch", "touch", "/tmp/pre_start_hook"}}, nil +} + +func main() { + fmt.Println(ContainerPlugin, DaemonPlugin) +} +``` +then build it with command line like: +``` +go build -buildmode=plugin -ldflags "-pluginpath=plugins_$(date +%s)" -o hook_plugin.so +``` +to use the shared object file generated, start pouchd which falg `--plugin=path_to_hook_plugin.so`, then when you start, stop daemon and create container in the log there will be some logs like: +``` +pre-start hook in daemon is called +pre create method called +pre-stop hook in daemon is called +``` +when you start a container, the config.json file (whose place is $home_dir/containerd/state/io.containerd.runtime.v1.linux/default/$container_id/config.json) will contains the pre-start hook you specified in your code, eg: +``` + "hooks": { + "prestart": [ + { + "args": [ + "libnetwork-setkey", + "f67df14e96fa4b94a6e386d0795bdd2703ca7b01713d48c9567203a37b05ae3d", + "8e3d8db7f72a66edee99d4db6ab911f8d618af057485731e9acf24b3668e25b6" + ], + "path": "/usr/local/bin/pouchd" + }, + { + "args": [ + "touch", + "/tmp/pre_start_hook" + ], + "path": "/usr/bin/touch" + } + ] + } +``` + +and if you use the exact code above, after starting a container the file at /tmp/pre_start_hook will be touched. + +## usage + +at pre-start daemon point you can start assit processes like network plugins and dfget proxy which need by pouchd and life cycle is the same as pouchd. +at pre-stop daemon point you can stop the assist processes gracefully, but this is point is not a promise, because pouchd may be killed by SIGKILL. +at pre-create container point you can change the input stream by some rule, in some company they have some stale orchestration system who use env to pass-in some limit which is an attribute in pouch, then you can use this point to convert value in env to attribute in ContainerConfig or HostConfig. +at pre-start container point you can set more pre-start hooks to oci spec, where you can do some special thing before container entrypoint start, priority decide the order of executing the hook. libnetwork hook has priority 0, so if the hook is expected to run before network in container setup you should set priority to a value big then 0, and vice versa. \ No newline at end of file diff --git a/internal/generator.go b/internal/generator.go index 31601adfb..66ff9d6d3 100644 --- a/internal/generator.go +++ b/internal/generator.go @@ -4,6 +4,7 @@ import ( "context" "path" + "github.com/alibaba/pouch/apis/plugins" "github.com/alibaba/pouch/ctrd" "github.com/alibaba/pouch/daemon/config" "github.com/alibaba/pouch/daemon/mgr" @@ -19,11 +20,12 @@ type DaemonProvider interface { VolMgr() mgr.VolumeMgr NetMgr() mgr.NetworkMgr MetaStore() *meta.Store + ContainerPlugin() plugins.ContainerPlugin } // GenContainerMgr generates a ContainerMgr instance according to config cfg. func GenContainerMgr(ctx context.Context, d DaemonProvider) (mgr.ContainerMgr, error) { - return mgr.NewContainerManager(ctx, d.MetaStore(), d.Containerd(), d.ImgMgr(), d.VolMgr(), d.NetMgr(), d.Config()) + return mgr.NewContainerManager(ctx, d.MetaStore(), d.Containerd(), d.ImgMgr(), d.VolMgr(), d.NetMgr(), d.Config(), d.ContainerPlugin()) } // GenSystemMgr generates a SystemMgr instance according to config cfg. diff --git a/main.go b/main.go index e6339fdf7..f7ba32204 100644 --- a/main.go +++ b/main.go @@ -88,6 +88,7 @@ func setupFlags(cmd *cobra.Command) { // cgroup-path flag is to set parent cgroup for all containers, default is "default" staying with containerd's configuration. flagSet.StringVar(&cfg.CgroupParent, "cgroup-parent", "default", "Set parent cgroup for all containers") + flagSet.StringVar(&cfg.PluginPath, "plugin", "", "Set the path where plugin shared library file put") } // parse flags @@ -194,7 +195,7 @@ func runDaemon() error { return fmt.Errorf("failed to new daemon") } - sigHandles = append(sigHandles, d.Shutdown) + sigHandles = append(sigHandles, d.ShutdownPlugin, d.Shutdown) return d.Run() }