From 08c74401c09357052491bfe306e2d7e9eeb0327d Mon Sep 17 00:00:00 2001 From: tonic Date: Thu, 7 Jul 2016 15:38:24 +0800 Subject: [PATCH] add artifacts retriever --- cluster/calcium/build_image.go | 13 ++- cluster/calcium/build_image_test.go | 4 +- cluster/calcium/cluster.go | 10 +- cluster/calcium/meta_test.go | 10 +- cluster/cluster.go | 2 +- devtools/client.py | 6 +- devtools/core_pb2.py | 51 +++++---- git/git.go | 77 -------------- network/calico/plugin.go | 14 ++- rpc/gen/core.pb.go | 146 +++++++++++++------------- rpc/gen/core.proto | 1 + rpc/rpc.go | 2 +- scheduler/simple/magnesium.go | 10 +- scheduler/simple/magnesium_test.go | 6 +- source/gitlab/cesium.go | 157 ++++++++++++++++++++++++++++ source/source.go | 8 ++ types/config.go | 5 +- 17 files changed, 324 insertions(+), 198 deletions(-) delete mode 100644 git/git.go create mode 100644 source/gitlab/cesium.go create mode 100644 source/source.go diff --git a/cluster/calcium/build_image.go b/cluster/calcium/build_image.go index 1c878d378..66c3335e1 100644 --- a/cluster/calcium/build_image.go +++ b/cluster/calcium/build_image.go @@ -11,7 +11,6 @@ import ( "github.com/docker/docker/pkg/archive" enginetypes "github.com/docker/engine-api/types" - "gitlab.ricebook.net/platform/core/git" "gitlab.ricebook.net/platform/core/types" "gitlab.ricebook.net/platform/core/utils" "golang.org/x/net/context" @@ -87,7 +86,7 @@ func getRandomNode(c *Calcium, podname string) (*types.Node, error) { // ├─ Dockerfile // ├─ launcher // ├─ launcheroot -func (c *Calcium) BuildImage(repository, version, uid string) (chan *types.BuildImageMessage, error) { +func (c *Calcium) BuildImage(repository, version, uid, artifact string) (chan *types.BuildImageMessage, error) { ch := make(chan *types.BuildImageMessage) // use pod `dev` to build image @@ -112,10 +111,18 @@ func (c *Calcium) BuildImage(repository, version, uid string) (chan *types.Build // clone code into cloneDir // which is under buildDir and named as repository name cloneDir := filepath.Join(buildDir, reponame) - if err := git.CloneRepository(repository, cloneDir, version, c.config.Git.PublicKey, c.config.Git.PrivateKey); err != nil { + if err := c.source.SourceCode(repository, cloneDir, version); err != nil { return ch, err } + // get artifact into cloneDir, only when artifact is not empty + // TODO need to separate from code or just like this? + if artifact != "" { + if err := c.source.Artifact(artifact, cloneDir); err != nil { + return ch, err + } + } + // ensure .git directory is removed // we don't want any history files to be retrieved if err := os.RemoveAll(filepath.Join(cloneDir, ".git")); err != nil { diff --git a/cluster/calcium/build_image_test.go b/cluster/calcium/build_image_test.go index 70e6d806c..2e6073095 100644 --- a/cluster/calcium/build_image_test.go +++ b/cluster/calcium/build_image_test.go @@ -6,13 +6,15 @@ import ( "github.com/stretchr/testify/assert" "gitlab.ricebook.net/platform/core/network/calico" "gitlab.ricebook.net/platform/core/scheduler/simple" + "gitlab.ricebook.net/platform/core/source/gitlab" "gitlab.ricebook.net/platform/core/store/mock" "gitlab.ricebook.net/platform/core/types" ) func TestGetRandomNode(t *testing.T) { store := &mockstore.MockStore{} - c := &Calcium{store: store, config: types.Config{}, scheduler: &simplescheduler.Magnesium{}, network: &calico.Titanium{}} + config := types.Config{} + c := &Calcium{store: store, config: config, scheduler: simplescheduler.New(), network: calico.New(), source: gitlab.New(config)} n1 := &types.Node{Name: "node1", Podname: "podname", Endpoint: "tcp://10.0.0.1:2376", CPU: types.CPUMap{"0": 10, "1": 10}} n2 := &types.Node{Name: "node2", Podname: "podname", Endpoint: "tcp://10.0.0.2:2376", CPU: types.CPUMap{"0": 10, "1": 10}} diff --git a/cluster/calcium/cluster.go b/cluster/calcium/cluster.go index 569ec3888..80edd10da 100644 --- a/cluster/calcium/cluster.go +++ b/cluster/calcium/cluster.go @@ -7,6 +7,8 @@ import ( "gitlab.ricebook.net/platform/core/network/calico" "gitlab.ricebook.net/platform/core/scheduler" "gitlab.ricebook.net/platform/core/scheduler/simple" + "gitlab.ricebook.net/platform/core/source" + "gitlab.ricebook.net/platform/core/source/gitlab" "gitlab.ricebook.net/platform/core/store" "gitlab.ricebook.net/platform/core/store/etcd" "gitlab.ricebook.net/platform/core/types" @@ -18,6 +20,7 @@ type Calcium struct { config types.Config scheduler scheduler.Scheduler network network.Network + source source.Source } func New(config types.Config) (*Calcium, error) { @@ -26,8 +29,9 @@ func New(config types.Config) (*Calcium, error) { return nil, err } - scheduler := &simplescheduler.Magnesium{} - titanium := &calico.Titanium{} + scheduler := simplescheduler.New() + titanium := calico.New() + source := gitlab.New(config) - return &Calcium{store: store, config: config, scheduler: scheduler, network: titanium}, nil + return &Calcium{store: store, config: config, scheduler: scheduler, network: titanium, source: source}, nil } diff --git a/cluster/calcium/meta_test.go b/cluster/calcium/meta_test.go index eb7a2dc18..4804f7518 100644 --- a/cluster/calcium/meta_test.go +++ b/cluster/calcium/meta_test.go @@ -7,13 +7,15 @@ import ( "github.com/stretchr/testify/assert" "gitlab.ricebook.net/platform/core/network/calico" "gitlab.ricebook.net/platform/core/scheduler/simple" + "gitlab.ricebook.net/platform/core/source/gitlab" "gitlab.ricebook.net/platform/core/store/mock" "gitlab.ricebook.net/platform/core/types" ) func TestListPods(t *testing.T) { store := &mockstore.MockStore{} - c := &Calcium{store: store, config: types.Config{}, scheduler: &simplescheduler.Magnesium{}, network: &calico.Titanium{}} + config := types.Config{} + c := &Calcium{store: store, config: config, scheduler: simplescheduler.New(), network: calico.New(), source: gitlab.New(config)} store.On("GetAllPods").Return([]*types.Pod{ &types.Pod{Name: "pod1", Desc: "desc1"}, @@ -33,7 +35,8 @@ func TestListPods(t *testing.T) { func TestAddPod(t *testing.T) { store := &mockstore.MockStore{} - c := &Calcium{store: store, config: types.Config{}, scheduler: &simplescheduler.Magnesium{}, network: &calico.Titanium{}} + config := types.Config{} + c := &Calcium{store: store, config: config, scheduler: simplescheduler.New(), network: calico.New(), source: gitlab.New(config)} store.On("AddPod", "pod1", "desc1").Return(&types.Pod{Name: "pod1", Desc: "desc1"}, nil) store.On("AddPod", "pod2", "desc2").Return(nil, fmt.Errorf("Etcd Error")) @@ -50,7 +53,8 @@ func TestAddPod(t *testing.T) { func TestGetPods(t *testing.T) { store := &mockstore.MockStore{} - c := &Calcium{store: store, config: types.Config{}, scheduler: &simplescheduler.Magnesium{}, network: &calico.Titanium{}} + config := types.Config{} + c := &Calcium{store: store, config: config, scheduler: simplescheduler.New(), network: calico.New(), source: gitlab.New(config)} store.On("GetPod", "pod1").Return(&types.Pod{Name: "pod1", Desc: "desc1"}, nil).Once() store.On("GetPod", "pod2").Return(nil, fmt.Errorf("Not found")).Once() diff --git a/cluster/cluster.go b/cluster/cluster.go index aa8ded25a..0c3d591d7 100644 --- a/cluster/cluster.go +++ b/cluster/cluster.go @@ -16,7 +16,7 @@ type Cluster interface { GetContainers(ids []string) ([]*types.Container, error) // cluster methods - BuildImage(repository, version, uid string) (chan *types.BuildImageMessage, error) + BuildImage(repository, version, uid, artifact string) (chan *types.BuildImageMessage, error) CreateContainer(specs types.Specs, opts *types.DeployOptions) (chan *types.CreateContainerMessage, error) UpgradeContainer(ids []string, image string) (chan *types.UpgradeContainerMessage, error) RemoveContainer(ids []string) (chan *types.RemoveContainerMessage, error) diff --git a/devtools/client.py b/devtools/client.py index 1272b27dd..d8845b590 100755 --- a/devtools/client.py +++ b/devtools/client.py @@ -137,10 +137,12 @@ def add_node(ctx, nodename, endpoint, podname, public): @click.argument('repo') @click.argument('version') @click.argument('uid') +@click.option('artifact', default='') @click.pass_context -def build_image(ctx, repo, version, uid): +def build_image(ctx, repo, version, uid, artifact): + # artifact = 'http://gitlab.ricebook.net/api/v3/projects/245/builds/1815/artifacts' stub = _get_stub(ctx) - opts = pb.BuildImageOptions(repo=repo, version=version, uid=uid) + opts = pb.BuildImageOptions(repo=repo, version=version, uid=uid, artifact=artifact) try: for m in stub.BuildImage(opts, 3600): diff --git a/devtools/core_pb2.py b/devtools/core_pb2.py index 12ff0c06b..35395943c 100644 --- a/devtools/core_pb2.py +++ b/devtools/core_pb2.py @@ -19,7 +19,7 @@ name='core.proto', package='pb', syntax='proto3', - serialized_pb=_b('\n\ncore.proto\x12\x02pb\"\x07\n\x05\x45mpty\"!\n\x03Pod\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04\x64\x65sc\x18\x02 \x01(\t\"\x1d\n\x04Pods\x12\x15\n\x04pods\x18\x01 \x03(\x0b\x32\x07.pb.Pod\"\x93\x01\n\x04Node\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08\x65ndpoint\x18\x02 \x01(\t\x12\x0f\n\x07podname\x18\x03 \x01(\t\x12\x0e\n\x06public\x18\x04 \x01(\x08\x12\x1e\n\x03\x63pu\x18\x05 \x03(\x0b\x32\x11.pb.Node.CpuEntry\x1a*\n\x08\x43puEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x03:\x02\x38\x01\" \n\x05Nodes\x12\x17\n\x05nodes\x18\x01 \x03(\x0b\x32\x08.pb.Node\"V\n\tContainer\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0f\n\x07podname\x18\x02 \x01(\t\x12\x10\n\x08nodename\x18\x03 \x01(\t\x12\x0c\n\x04name\x18\x04 \x01(\t\x12\x0c\n\x04info\x18\x05 \x01(\t\"/\n\nContainers\x12!\n\ncontainers\x18\x01 \x03(\x0b\x32\r.pb.Container\"\x19\n\x0b\x43ontainerID\x12\n\n\x02id\x18\x01 \x01(\t\",\n\x0c\x43ontainerIDs\x12\x1c\n\x03ids\x18\x01 \x03(\x0b\x32\x0f.pb.ContainerID\"+\n\rAddPodOptions\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04\x64\x65sc\x18\x02 \x01(\t\"\x1d\n\rGetPodOptions\x12\x0c\n\x04name\x18\x01 \x01(\t\"U\n\x0e\x41\x64\x64NodeOptions\x12\x10\n\x08nodename\x18\x01 \x01(\t\x12\x10\n\x08\x65ndpoint\x18\x02 \x01(\t\x12\x0f\n\x07podname\x18\x03 \x01(\t\x12\x0e\n\x06public\x18\x04 \x01(\x08\"3\n\x0eGetNodeOptions\x12\x0f\n\x07podname\x18\x01 \x01(\t\x12\x10\n\x08nodename\x18\x02 \x01(\t\"#\n\x10ListNodesOptions\x12\x0f\n\x07podname\x18\x01 \x01(\t\"?\n\x11\x42uildImageOptions\x12\x0c\n\x04repo\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x0b\n\x03uid\x18\x03 \x01(\t\"\x95\x02\n\rDeployOptions\x12\r\n\x05specs\x18\x01 \x01(\t\x12\x0f\n\x07\x61ppname\x18\x02 \x01(\t\x12\r\n\x05image\x18\x03 \x01(\t\x12\x0f\n\x07podname\x18\x04 \x01(\t\x12\x10\n\x08nodename\x18\x05 \x01(\t\x12\x12\n\nentrypoint\x18\x06 \x01(\t\x12\x11\n\tcpu_quota\x18\x07 \x01(\x01\x12\r\n\x05\x63ount\x18\x08 \x01(\x05\x12\x0b\n\x03\x65nv\x18\t \x03(\t\x12\x31\n\x08networks\x18\n \x03(\x0b\x32\x1f.pb.DeployOptions.NetworksEntry\x12\x0b\n\x03raw\x18\x0b \x01(\x08\x1a/\n\rNetworksEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"=\n\x0eUpgradeOptions\x12\x1c\n\x03ids\x18\x01 \x03(\x0b\x32\x0f.pb.ContainerID\x12\r\n\x05image\x18\x02 \x01(\t\"G\n\x12RemoveImageOptions\x12\x0f\n\x07podname\x18\x01 \x01(\t\x12\x10\n\x08nodename\x18\x02 \x01(\t\x12\x0e\n\x06images\x18\x03 \x03(\t\",\n\x0b\x45rrorDetail\x12\x0c\n\x04\x63ode\x18\x01 \x01(\x03\x12\x0f\n\x07message\x18\x02 \x01(\t\"{\n\x11\x42uildImageMessage\x12\x0e\n\x06status\x18\x01 \x01(\t\x12\x10\n\x08progress\x18\x02 \x01(\t\x12\r\n\x05\x65rror\x18\x03 \x01(\t\x12\x0e\n\x06stream\x18\x04 \x01(\t\x12%\n\x0c\x65rror_detail\x18\x05 \x01(\x0b\x32\x0f.pb.ErrorDetail\"\xd3\x01\n\x16\x43reateContainerMessage\x12\x0f\n\x07podname\x18\x01 \x01(\t\x12\x10\n\x08nodename\x18\x02 \x01(\t\x12\n\n\x02id\x18\x03 \x01(\t\x12\x0c\n\x04name\x18\x04 \x01(\t\x12\r\n\x05\x65rror\x18\x05 \x01(\t\x12\x0f\n\x07success\x18\x06 \x01(\x08\x12\x30\n\x03\x63pu\x18\x07 \x03(\x0b\x32#.pb.CreateContainerMessage.CpuEntry\x1a*\n\x08\x43puEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x03:\x02\x38\x01\"F\n\x12RemoveImageMessage\x12\r\n\x05image\x18\x01 \x01(\t\x12\x0f\n\x07success\x18\x02 \x01(\x08\x12\x10\n\x08messages\x18\x03 \x03(\t\"F\n\x16RemoveContainerMessage\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0f\n\x07success\x18\x02 \x01(\x08\x12\x0f\n\x07message\x18\x03 \x01(\t\"g\n\x17UpgradeContainerMessage\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0e\n\x06new_id\x18\x02 \x01(\t\x12\x10\n\x08new_name\x18\x03 \x01(\t\x12\r\n\x05\x65rror\x18\x04 \x01(\t\x12\x0f\n\x07success\x18\x05 \x01(\x08\x32\xbb\x05\n\x07\x43oreRPC\x12!\n\x08ListPods\x12\t.pb.Empty\x1a\x08.pb.Pods\"\x00\x12&\n\x06\x41\x64\x64Pod\x12\x11.pb.AddPodOptions\x1a\x07.pb.Pod\"\x00\x12&\n\x06GetPod\x12\x11.pb.GetPodOptions\x1a\x07.pb.Pod\"\x00\x12)\n\x07\x41\x64\x64Node\x12\x12.pb.AddNodeOptions\x1a\x08.pb.Node\"\x00\x12)\n\x07GetNode\x12\x12.pb.GetNodeOptions\x1a\x08.pb.Node\"\x00\x12\x31\n\x0cListPodNodes\x12\x14.pb.ListNodesOptions\x1a\t.pb.Nodes\"\x00\x12\x30\n\x0cGetContainer\x12\x0f.pb.ContainerID\x1a\r.pb.Container\"\x00\x12\x33\n\rGetContainers\x12\x10.pb.ContainerIDs\x1a\x0e.pb.Containers\"\x00\x12>\n\nBuildImage\x12\x15.pb.BuildImageOptions\x1a\x15.pb.BuildImageMessage\"\x00\x30\x01\x12\x42\n\x0f\x43reateContainer\x12\x11.pb.DeployOptions\x1a\x1a.pb.CreateContainerMessage0\x01\x12\x45\n\x10UpgradeContainer\x12\x12.pb.UpgradeOptions\x1a\x1b.pb.UpgradeContainerMessage0\x01\x12\x41\n\x0fRemoveContainer\x12\x10.pb.ContainerIDs\x1a\x1a.pb.RemoveContainerMessage0\x01\x12?\n\x0bRemoveImage\x12\x16.pb.RemoveImageOptions\x1a\x16.pb.RemoveImageMessage0\x01\x62\x06proto3') + serialized_pb=_b('\n\ncore.proto\x12\x02pb\"\x07\n\x05\x45mpty\"!\n\x03Pod\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04\x64\x65sc\x18\x02 \x01(\t\"\x1d\n\x04Pods\x12\x15\n\x04pods\x18\x01 \x03(\x0b\x32\x07.pb.Pod\"\x93\x01\n\x04Node\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08\x65ndpoint\x18\x02 \x01(\t\x12\x0f\n\x07podname\x18\x03 \x01(\t\x12\x0e\n\x06public\x18\x04 \x01(\x08\x12\x1e\n\x03\x63pu\x18\x05 \x03(\x0b\x32\x11.pb.Node.CpuEntry\x1a*\n\x08\x43puEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x03:\x02\x38\x01\" \n\x05Nodes\x12\x17\n\x05nodes\x18\x01 \x03(\x0b\x32\x08.pb.Node\"V\n\tContainer\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0f\n\x07podname\x18\x02 \x01(\t\x12\x10\n\x08nodename\x18\x03 \x01(\t\x12\x0c\n\x04name\x18\x04 \x01(\t\x12\x0c\n\x04info\x18\x05 \x01(\t\"/\n\nContainers\x12!\n\ncontainers\x18\x01 \x03(\x0b\x32\r.pb.Container\"\x19\n\x0b\x43ontainerID\x12\n\n\x02id\x18\x01 \x01(\t\",\n\x0c\x43ontainerIDs\x12\x1c\n\x03ids\x18\x01 \x03(\x0b\x32\x0f.pb.ContainerID\"+\n\rAddPodOptions\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04\x64\x65sc\x18\x02 \x01(\t\"\x1d\n\rGetPodOptions\x12\x0c\n\x04name\x18\x01 \x01(\t\"U\n\x0e\x41\x64\x64NodeOptions\x12\x10\n\x08nodename\x18\x01 \x01(\t\x12\x10\n\x08\x65ndpoint\x18\x02 \x01(\t\x12\x0f\n\x07podname\x18\x03 \x01(\t\x12\x0e\n\x06public\x18\x04 \x01(\x08\"3\n\x0eGetNodeOptions\x12\x0f\n\x07podname\x18\x01 \x01(\t\x12\x10\n\x08nodename\x18\x02 \x01(\t\"#\n\x10ListNodesOptions\x12\x0f\n\x07podname\x18\x01 \x01(\t\"Q\n\x11\x42uildImageOptions\x12\x0c\n\x04repo\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x0b\n\x03uid\x18\x03 \x01(\t\x12\x10\n\x08\x61rtifact\x18\x04 \x01(\t\"\x95\x02\n\rDeployOptions\x12\r\n\x05specs\x18\x01 \x01(\t\x12\x0f\n\x07\x61ppname\x18\x02 \x01(\t\x12\r\n\x05image\x18\x03 \x01(\t\x12\x0f\n\x07podname\x18\x04 \x01(\t\x12\x10\n\x08nodename\x18\x05 \x01(\t\x12\x12\n\nentrypoint\x18\x06 \x01(\t\x12\x11\n\tcpu_quota\x18\x07 \x01(\x01\x12\r\n\x05\x63ount\x18\x08 \x01(\x05\x12\x0b\n\x03\x65nv\x18\t \x03(\t\x12\x31\n\x08networks\x18\n \x03(\x0b\x32\x1f.pb.DeployOptions.NetworksEntry\x12\x0b\n\x03raw\x18\x0b \x01(\x08\x1a/\n\rNetworksEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"=\n\x0eUpgradeOptions\x12\x1c\n\x03ids\x18\x01 \x03(\x0b\x32\x0f.pb.ContainerID\x12\r\n\x05image\x18\x02 \x01(\t\"G\n\x12RemoveImageOptions\x12\x0f\n\x07podname\x18\x01 \x01(\t\x12\x10\n\x08nodename\x18\x02 \x01(\t\x12\x0e\n\x06images\x18\x03 \x03(\t\",\n\x0b\x45rrorDetail\x12\x0c\n\x04\x63ode\x18\x01 \x01(\x03\x12\x0f\n\x07message\x18\x02 \x01(\t\"{\n\x11\x42uildImageMessage\x12\x0e\n\x06status\x18\x01 \x01(\t\x12\x10\n\x08progress\x18\x02 \x01(\t\x12\r\n\x05\x65rror\x18\x03 \x01(\t\x12\x0e\n\x06stream\x18\x04 \x01(\t\x12%\n\x0c\x65rror_detail\x18\x05 \x01(\x0b\x32\x0f.pb.ErrorDetail\"\xd3\x01\n\x16\x43reateContainerMessage\x12\x0f\n\x07podname\x18\x01 \x01(\t\x12\x10\n\x08nodename\x18\x02 \x01(\t\x12\n\n\x02id\x18\x03 \x01(\t\x12\x0c\n\x04name\x18\x04 \x01(\t\x12\r\n\x05\x65rror\x18\x05 \x01(\t\x12\x0f\n\x07success\x18\x06 \x01(\x08\x12\x30\n\x03\x63pu\x18\x07 \x03(\x0b\x32#.pb.CreateContainerMessage.CpuEntry\x1a*\n\x08\x43puEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x03:\x02\x38\x01\"F\n\x12RemoveImageMessage\x12\r\n\x05image\x18\x01 \x01(\t\x12\x0f\n\x07success\x18\x02 \x01(\x08\x12\x10\n\x08messages\x18\x03 \x03(\t\"F\n\x16RemoveContainerMessage\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0f\n\x07success\x18\x02 \x01(\x08\x12\x0f\n\x07message\x18\x03 \x01(\t\"g\n\x17UpgradeContainerMessage\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0e\n\x06new_id\x18\x02 \x01(\t\x12\x10\n\x08new_name\x18\x03 \x01(\t\x12\r\n\x05\x65rror\x18\x04 \x01(\t\x12\x0f\n\x07success\x18\x05 \x01(\x08\x32\xbb\x05\n\x07\x43oreRPC\x12!\n\x08ListPods\x12\t.pb.Empty\x1a\x08.pb.Pods\"\x00\x12&\n\x06\x41\x64\x64Pod\x12\x11.pb.AddPodOptions\x1a\x07.pb.Pod\"\x00\x12&\n\x06GetPod\x12\x11.pb.GetPodOptions\x1a\x07.pb.Pod\"\x00\x12)\n\x07\x41\x64\x64Node\x12\x12.pb.AddNodeOptions\x1a\x08.pb.Node\"\x00\x12)\n\x07GetNode\x12\x12.pb.GetNodeOptions\x1a\x08.pb.Node\"\x00\x12\x31\n\x0cListPodNodes\x12\x14.pb.ListNodesOptions\x1a\t.pb.Nodes\"\x00\x12\x30\n\x0cGetContainer\x12\x0f.pb.ContainerID\x1a\r.pb.Container\"\x00\x12\x33\n\rGetContainers\x12\x10.pb.ContainerIDs\x1a\x0e.pb.Containers\"\x00\x12>\n\nBuildImage\x12\x15.pb.BuildImageOptions\x1a\x15.pb.BuildImageMessage\"\x00\x30\x01\x12\x42\n\x0f\x43reateContainer\x12\x11.pb.DeployOptions\x1a\x1a.pb.CreateContainerMessage0\x01\x12\x45\n\x10UpgradeContainer\x12\x12.pb.UpgradeOptions\x1a\x1b.pb.UpgradeContainerMessage0\x01\x12\x41\n\x0fRemoveContainer\x12\x10.pb.ContainerIDs\x1a\x1a.pb.RemoveContainerMessage0\x01\x12?\n\x0bRemoveImage\x12\x16.pb.RemoveImageOptions\x1a\x16.pb.RemoveImageMessage0\x01\x62\x06proto3') ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -616,6 +616,13 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + _descriptor.FieldDescriptor( + name='artifact', full_name='pb.BuildImageOptions.artifact', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -629,7 +636,7 @@ oneofs=[ ], serialized_start=740, - serialized_end=803, + serialized_end=821, ) @@ -666,8 +673,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1036, - serialized_end=1083, + serialized_start=1054, + serialized_end=1101, ) _DEPLOYOPTIONS = _descriptor.Descriptor( @@ -766,8 +773,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=806, - serialized_end=1083, + serialized_start=824, + serialized_end=1101, ) @@ -804,8 +811,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1085, - serialized_end=1146, + serialized_start=1103, + serialized_end=1164, ) @@ -849,8 +856,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1148, - serialized_end=1219, + serialized_start=1166, + serialized_end=1237, ) @@ -887,8 +894,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1221, - serialized_end=1265, + serialized_start=1239, + serialized_end=1283, ) @@ -946,8 +953,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1267, - serialized_end=1390, + serialized_start=1285, + serialized_end=1408, ) @@ -1056,8 +1063,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1393, - serialized_end=1604, + serialized_start=1411, + serialized_end=1622, ) @@ -1101,8 +1108,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1606, - serialized_end=1676, + serialized_start=1624, + serialized_end=1694, ) @@ -1146,8 +1153,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1678, - serialized_end=1748, + serialized_start=1696, + serialized_end=1766, ) @@ -1205,8 +1212,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1750, - serialized_end=1853, + serialized_start=1768, + serialized_end=1871, ) _PODS.fields_by_name['pods'].message_type = _POD diff --git a/git/git.go b/git/git.go deleted file mode 100644 index 9b40b47f9..000000000 --- a/git/git.go +++ /dev/null @@ -1,77 +0,0 @@ -package git - -import ( - "fmt" - "os" - "strings" - - "gopkg.in/libgit2/git2go.v23" -) - -const gitUser = "git" - -func certificateCheckCallback(cert *git.Certificate, valid bool, hostname string) git.ErrorCode { - return git.ErrorCode(0) -} - -// clone `repository` to `path` and checkout `revision` -// using pubkey and prikey -func CloneRepository(repository, path, revision, pubkey, prikey string) error { - if !strings.HasPrefix(repository, "git@") { - return fmt.Errorf("Only support ssh protocol(%q), use git@...", repository) - } - if _, err := os.Stat(pubkey); os.IsNotExist(err) { - return fmt.Errorf("Public Key not found(%q)", pubkey) - } - if _, err := os.Stat(prikey); os.IsNotExist(err) { - return fmt.Errorf("Private Key not found(%q)", prikey) - } - - credentialsCallback := func(url, username string, allowedTypes git.CredType) (git.ErrorCode, *git.Cred) { - ret, cred := git.NewCredSshKey(gitUser, pubkey, prikey, "") - return git.ErrorCode(ret), &cred - } - - cloneOpts := &git.CloneOptions{ - FetchOptions: &git.FetchOptions{ - RemoteCallbacks: git.RemoteCallbacks{ - CredentialsCallback: credentialsCallback, - CertificateCheckCallback: certificateCheckCallback, - }, - }, - } - - repo, err := git.Clone(repository, path, cloneOpts) - if err != nil { - return err - } - - if err := repo.CheckoutHead(nil); err != nil { - return err - } - - object, err := repo.RevparseSingle(revision) - if err != nil { - return err - } - defer object.Free() - - // below is code for v24 - // which is fucking unstable - // ------------------------- - // - // tree, err := object.AsTree() - // if err != nil { - // return err - // } - obj, err := object.Peel(git.ObjectTree) - if err != nil { - return err - } - - tree, ok := obj.(*git.Tree) - if !ok { - return fmt.Errorf("git object is not a tree") - } - return repo.CheckoutTree(tree, &git.CheckoutOpts{Strategy: git.CheckoutSafe}) -} diff --git a/network/calico/plugin.go b/network/calico/plugin.go index c7a6f3587..7440fdaf1 100644 --- a/network/calico/plugin.go +++ b/network/calico/plugin.go @@ -10,23 +10,23 @@ import ( "golang.org/x/net/context" ) -type Titanium struct{} +type titanium struct{} // type of the network manager // if set to "plugin", then it will act like a plugin // if set to "agent", then it will act like an agent // main difference is the order of connect/disconnect -func (t *Titanium) Type() string { +func (t *titanium) Type() string { return "plugin" } // name of the network manager -func (t *Titanium) Name() string { +func (t *titanium) Name() string { return "calico" } // connect to network with ipv4 address -func (t *Titanium) ConnectToNetwork(ctx context.Context, containerID, networkID, ipv4 string) error { +func (t *titanium) ConnectToNetwork(ctx context.Context, containerID, networkID, ipv4 string) error { if len(containerID) != 64 { return fmt.Errorf("ContainerID must be in length of 64") } @@ -56,7 +56,7 @@ func (t *Titanium) ConnectToNetwork(ctx context.Context, containerID, networkID, } // disconnect from network -func (t *Titanium) DisconnectFromNetwork(ctx context.Context, containerID, networkID string) error { +func (t *titanium) DisconnectFromNetwork(ctx context.Context, containerID, networkID string) error { if len(containerID) != 64 { return fmt.Errorf("ContainerID must be in length of 64") } @@ -69,3 +69,7 @@ func (t *Titanium) DisconnectFromNetwork(ctx context.Context, containerID, netwo log.Debugf("Disconnect %q from %q", containerID, networkID) return engine.NetworkDisconnect(context.Background(), networkID, containerID, false) } + +func New() *titanium { + return &titanium{} +} diff --git a/rpc/gen/core.pb.go b/rpc/gen/core.pb.go index d6f108f70..29b9df0fb 100644 --- a/rpc/gen/core.pb.go +++ b/rpc/gen/core.pb.go @@ -234,9 +234,10 @@ func (*ListNodesOptions) ProtoMessage() {} func (*ListNodesOptions) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} } type BuildImageOptions struct { - Repo string `protobuf:"bytes,1,opt,name=repo" json:"repo,omitempty"` - Version string `protobuf:"bytes,2,opt,name=version" json:"version,omitempty"` - Uid string `protobuf:"bytes,3,opt,name=uid" json:"uid,omitempty"` + Repo string `protobuf:"bytes,1,opt,name=repo" json:"repo,omitempty"` + Version string `protobuf:"bytes,2,opt,name=version" json:"version,omitempty"` + Uid string `protobuf:"bytes,3,opt,name=uid" json:"uid,omitempty"` + Artifact string `protobuf:"bytes,4,opt,name=artifact" json:"artifact,omitempty"` } func (m *BuildImageOptions) Reset() { *m = BuildImageOptions{} } @@ -1018,74 +1019,75 @@ var _CoreRPC_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("core.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 1095 bytes of a gzipped FileDescriptorProto + // 1107 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x57, 0x6d, 0x6f, 0xdc, 0x44, - 0x10, 0xae, 0xed, 0xf3, 0xbd, 0xcc, 0x35, 0x2f, 0x5d, 0xb5, 0xe1, 0x70, 0x45, 0x81, 0x8d, 0x04, - 0x45, 0xa2, 0x51, 0x49, 0xc5, 0x8b, 0x88, 0x04, 0x6a, 0x93, 0x80, 0x22, 0x41, 0x28, 0x46, 0x7c, - 0x43, 0x0a, 0xce, 0x79, 0x89, 0xac, 0xe6, 0x6c, 0xe3, 0x97, 0x44, 0xe1, 0x2f, 0xf0, 0x89, 0x9f, - 0xc1, 0x6f, 0xe0, 0x33, 0xff, 0x8a, 0x0f, 0xcc, 0xec, 0x8b, 0x6f, 0xd7, 0xbd, 0x10, 0x51, 0xf5, - 0xd3, 0xed, 0xcc, 0x8e, 0x9f, 0x9d, 0x99, 0x7d, 0x66, 0x67, 0x0e, 0x60, 0x5e, 0x54, 0x62, 0xa7, - 0xac, 0x8a, 0xa6, 0x60, 0x7e, 0x79, 0xca, 0x47, 0x10, 0x1e, 0x2e, 0xca, 0xe6, 0x8a, 0x3f, 0x82, - 0xe0, 0x79, 0x91, 0x32, 0x06, 0x83, 0x3c, 0x59, 0x88, 0x99, 0xf7, 0x8e, 0xf7, 0x70, 0x12, 0xcb, - 0x35, 0xe9, 0x52, 0x51, 0xcf, 0x67, 0xbe, 0xd2, 0xd1, 0x9a, 0x6f, 0xc3, 0x00, 0xcd, 0x6b, 0x76, - 0x1f, 0x06, 0x25, 0xfe, 0xa2, 0x7d, 0xf0, 0x70, 0xba, 0x3b, 0xda, 0x29, 0x4f, 0x77, 0x50, 0x1f, - 0x4b, 0x25, 0xff, 0xdb, 0x83, 0xc1, 0x71, 0x91, 0x8a, 0x95, 0xa8, 0x11, 0x8c, 0x45, 0x9e, 0x96, - 0x45, 0x96, 0x37, 0x1a, 0xb9, 0x93, 0xd9, 0x0c, 0x46, 0x08, 0x20, 0x3f, 0x09, 0xe4, 0x96, 0x11, - 0xd9, 0x16, 0x0c, 0xcb, 0xf6, 0xf4, 0x3c, 0x9b, 0xcf, 0x06, 0xb8, 0x31, 0x8e, 0xb5, 0xc4, 0xb6, - 0x21, 0x98, 0x97, 0xed, 0x2c, 0x94, 0x6e, 0xdc, 0x21, 0x37, 0xe8, 0xe0, 0x9d, 0xfd, 0xb2, 0x3d, - 0xcc, 0x9b, 0xea, 0x2a, 0xa6, 0xdd, 0xe8, 0x13, 0x18, 0x1b, 0x05, 0xdb, 0x84, 0xe0, 0x85, 0xb8, - 0xd2, 0x1e, 0xd1, 0x92, 0xdd, 0x85, 0xf0, 0x22, 0x39, 0x6f, 0x85, 0xf4, 0x26, 0x88, 0x95, 0xf0, - 0xb9, 0xff, 0x99, 0xc7, 0xdf, 0x87, 0x90, 0xd0, 0x6a, 0xf6, 0x00, 0xc2, 0x9c, 0x16, 0x3a, 0xdc, - 0xb1, 0x39, 0x27, 0x56, 0x6a, 0x7e, 0x05, 0x93, 0xfd, 0x22, 0x6f, 0x92, 0x2c, 0x17, 0x15, 0x5b, - 0x07, 0x3f, 0x4b, 0xf5, 0x01, 0xb8, 0xb2, 0x83, 0xf2, 0xdd, 0xa0, 0x30, 0x15, 0xf4, 0xbd, 0x15, - 0x6f, 0x27, 0x77, 0xa9, 0x1b, 0xb8, 0x17, 0x92, 0xe5, 0xbf, 0x14, 0x18, 0xad, 0xd4, 0xd1, 0x9a, - 0xef, 0x01, 0x74, 0x47, 0xd7, 0xec, 0x11, 0x5d, 0xb4, 0x91, 0xb4, 0xb7, 0x6b, 0xe4, 0x6d, 0x67, - 0x13, 0x5b, 0x06, 0xfc, 0x2d, 0x98, 0x76, 0x1b, 0x47, 0x07, 0x7d, 0xcf, 0xf9, 0x47, 0x70, 0xdb, - 0xda, 0xae, 0xd9, 0xbb, 0x10, 0x64, 0xdd, 0x9d, 0x6f, 0x38, 0xb0, 0x47, 0x07, 0x31, 0xed, 0xf1, - 0x4f, 0x61, 0xed, 0x69, 0x9a, 0x22, 0x15, 0xbe, 0x2b, 0x9b, 0xac, 0xc8, 0xeb, 0xff, 0x41, 0xac, - 0xb5, 0xaf, 0x45, 0xf3, 0xdf, 0x1f, 0xf2, 0xdf, 0x60, 0x1d, 0xd1, 0x29, 0xf3, 0xc6, 0xca, 0x4e, - 0xa1, 0xd7, 0x4b, 0xe1, 0x6b, 0x65, 0x1a, 0xff, 0x0a, 0xd6, 0xd1, 0x41, 0xfb, 0x6c, 0x0b, 0xc3, - 0xbb, 0xfe, 0x62, 0x7d, 0xd7, 0x2b, 0xfe, 0x21, 0x6c, 0x7e, 0x93, 0xd5, 0x12, 0xa8, 0xbe, 0x11, - 0x89, 0xff, 0x00, 0x77, 0x9e, 0xb5, 0xd9, 0x79, 0x7a, 0xb4, 0x48, 0xce, 0x84, 0x95, 0x9a, 0x4a, - 0x94, 0x85, 0x49, 0x0d, 0xad, 0x09, 0xe2, 0x02, 0xaf, 0x14, 0xf7, 0x0d, 0xcb, 0xb4, 0x48, 0x8c, - 0x6f, 0xf1, 0x5a, 0x55, 0x98, 0xb4, 0xe4, 0xff, 0xf8, 0xb0, 0x76, 0x20, 0xca, 0xf3, 0xe2, 0xca, - 0x20, 0x62, 0x0d, 0xd4, 0xa5, 0x98, 0xd7, 0x1a, 0x52, 0x09, 0x84, 0x99, 0x94, 0xa5, 0xcd, 0x5c, - 0x2d, 0x92, 0x7d, 0x46, 0x1e, 0x69, 0x54, 0x25, 0xd8, 0x61, 0x0c, 0xae, 0x4f, 0x48, 0xd8, 0xbb, - 0xa6, 0x07, 0x00, 0x82, 0x4a, 0x53, 0x5d, 0xd4, 0x50, 0xee, 0x5a, 0x1a, 0x7c, 0x6a, 0x26, 0x58, - 0xc4, 0x27, 0xbf, 0xb6, 0x45, 0x93, 0xcc, 0x46, 0xb8, 0xed, 0xc5, 0x63, 0x54, 0x7c, 0x4f, 0x32, - 0x39, 0x32, 0x2f, 0x5a, 0xfc, 0x6e, 0x8c, 0x1b, 0x61, 0xac, 0x04, 0x0a, 0x59, 0xe4, 0x17, 0xb3, - 0x09, 0x12, 0x15, 0x43, 0xc6, 0x25, 0xdb, 0x43, 0x07, 0x44, 0x73, 0x59, 0x54, 0x2f, 0xea, 0x19, - 0x48, 0xfe, 0xbe, 0x4d, 0xfc, 0x75, 0xb2, 0xb0, 0x73, 0xac, 0x2d, 0xd4, 0xd3, 0xd1, 0x7d, 0x40, - 0x70, 0x55, 0x72, 0x39, 0x9b, 0x4a, 0x3e, 0xd0, 0x32, 0xda, 0x83, 0x35, 0xc7, 0xf8, 0xa6, 0x67, - 0x65, 0x62, 0x3f, 0x2b, 0x47, 0xb0, 0xfe, 0x63, 0x79, 0x56, 0x25, 0x4b, 0x26, 0xdd, 0x5c, 0x58, - 0xcb, 0x8c, 0xfb, 0x56, 0xc6, 0xf9, 0x29, 0xb0, 0x58, 0x2c, 0x8a, 0x0b, 0xe1, 0xf0, 0xe3, 0x95, - 0x88, 0x49, 0xc4, 0x97, 0xa0, 0x35, 0x5e, 0x2a, 0xe5, 0x4d, 0x4b, 0xf8, 0xc2, 0x4c, 0x0f, 0xab, - 0xaa, 0xa8, 0x0e, 0x04, 0x7a, 0x74, 0x4e, 0xe4, 0x9b, 0xe3, 0x27, 0x12, 0x39, 0x88, 0xe5, 0x9a, - 0x0e, 0x5c, 0x88, 0xba, 0x5e, 0xba, 0x67, 0x44, 0xfe, 0xa7, 0x67, 0x13, 0xf8, 0x5b, 0xa5, 0xa5, - 0xa3, 0xea, 0x26, 0x69, 0x5a, 0xc3, 0x37, 0x2d, 0x91, 0x7b, 0xd8, 0xa2, 0xce, 0x2a, 0x34, 0x33, - 0xee, 0x19, 0x99, 0x12, 0x20, 0xc8, 0x0d, 0x43, 0x39, 0x29, 0x28, 0xa4, 0x4a, 0x24, 0x0b, 0xcd, - 0x38, 0x2d, 0xb1, 0x5d, 0xb8, 0x2d, 0x0d, 0x4e, 0x52, 0xe9, 0xb5, 0x24, 0x9d, 0x4e, 0xad, 0x15, - 0x4c, 0x3c, 0x15, 0x4b, 0x81, 0xff, 0xe1, 0xc3, 0xd6, 0x3e, 0x7e, 0xdd, 0x88, 0x2e, 0xfb, 0xc6, - 0xe1, 0x57, 0xcb, 0xa8, 0x7a, 0x4f, 0x83, 0xae, 0x13, 0xac, 0x7a, 0xd3, 0xbb, 0xb0, 0x42, 0x3b, - 0x2c, 0x3c, 0xaf, 0x6e, 0xe7, 0x73, 0xca, 0xc3, 0x50, 0xb2, 0xce, 0x88, 0xec, 0x63, 0xd5, 0xf0, - 0x46, 0x92, 0x2a, 0xdb, 0x92, 0x2a, 0x2b, 0x5d, 0x7e, 0x4d, 0x2d, 0xf0, 0x67, 0x87, 0x60, 0x26, - 0x1d, 0x1d, 0x19, 0xbd, 0x5e, 0xf9, 0x1b, 0xa7, 0x7d, 0xd7, 0x69, 0x4c, 0x92, 0x26, 0x84, 0x21, - 0x57, 0x27, 0xf3, 0x9f, 0x60, 0x4b, 0x9d, 0xf0, 0x52, 0xd2, 0x57, 0x34, 0xd2, 0x6b, 0xf0, 0x2d, - 0xfe, 0x05, 0x2e, 0xff, 0x7e, 0xf7, 0xe0, 0x0d, 0x5d, 0x6c, 0x37, 0xe2, 0xdf, 0x83, 0x61, 0x2e, - 0x2e, 0x4f, 0x50, 0xa7, 0x6b, 0x0c, 0xa5, 0xa3, 0x94, 0xbd, 0x49, 0x4f, 0xc7, 0xe5, 0x89, 0xdd, - 0x2b, 0x50, 0x3e, 0x76, 0x2e, 0x6f, 0x70, 0xcd, 0xe5, 0x85, 0x8e, 0x9f, 0xbb, 0x7f, 0x85, 0x30, - 0xda, 0xc7, 0x41, 0x2c, 0x7e, 0xbe, 0x8f, 0x35, 0x3f, 0xa6, 0x3e, 0x20, 0xa7, 0xa9, 0x89, 0xe4, - 0x25, 0xcd, 0x63, 0xd1, 0x58, 0x8f, 0x52, 0x35, 0xbf, 0xc5, 0xde, 0x83, 0xa1, 0x6a, 0xa6, 0x4c, - 0x4e, 0x36, 0x4e, 0x63, 0x8d, 0xcc, 0xcc, 0xa5, 0xec, 0x54, 0xef, 0x54, 0x76, 0x4e, 0x1f, 0xb5, - 0xed, 0x3e, 0x80, 0x91, 0x6e, 0x9f, 0x8c, 0x69, 0x40, 0xab, 0x9f, 0x45, 0xdd, 0x58, 0xa3, 0x4c, - 0x75, 0xb7, 0x53, 0xa6, 0x6e, 0xeb, 0x73, 0x4c, 0x71, 0x4a, 0xd0, 0x81, 0xa8, 0x61, 0xe9, 0x2e, - 0xed, 0xf5, 0x5b, 0x5c, 0x34, 0x31, 0x5f, 0x50, 0x60, 0x8f, 0xe1, 0x36, 0x02, 0x2e, 0x47, 0xa6, - 0xfe, 0x93, 0x17, 0xb9, 0x33, 0x0b, 0x7e, 0xf1, 0x44, 0x8e, 0x07, 0xd6, 0xa4, 0xb3, 0xd9, 0xfb, - 0xa4, 0x8e, 0xd6, 0x1d, 0x0d, 0x1d, 0xf3, 0x05, 0xc0, 0xf2, 0xed, 0x61, 0xf7, 0x68, 0xff, 0xa5, - 0x66, 0x1a, 0xf5, 0xd4, 0x9a, 0x1c, 0xfc, 0xd6, 0x63, 0x8f, 0x3d, 0x83, 0x8d, 0x5e, 0x71, 0xa9, - 0x04, 0x3b, 0x5d, 0x23, 0x8a, 0xae, 0x2f, 0x42, 0xc4, 0x38, 0x84, 0xcd, 0x3e, 0xff, 0x54, 0x46, - 0xdd, 0x16, 0x10, 0xdd, 0xb7, 0x74, 0x2b, 0x60, 0x9e, 0xc2, 0x46, 0xaf, 0x4a, 0x56, 0x64, 0x40, - 0x7a, 0xb2, 0xba, 0x98, 0x10, 0xe2, 0x4b, 0x98, 0x5a, 0xa5, 0xcc, 0xb6, 0x96, 0xc6, 0x4e, 0x3e, - 0xfa, 0xfa, 0x0e, 0xe0, 0x74, 0x28, 0xff, 0x3e, 0x3c, 0xf9, 0x37, 0x00, 0x00, 0xff, 0xff, 0x0d, - 0x15, 0x86, 0xce, 0x4c, 0x0c, 0x00, 0x00, + 0x10, 0xae, 0xcf, 0xe7, 0x7b, 0x99, 0x6b, 0x5e, 0xba, 0x6a, 0xc3, 0xe1, 0x8a, 0x02, 0x1b, 0x09, + 0x8a, 0x44, 0xa3, 0x92, 0x8a, 0x17, 0x11, 0x09, 0xd4, 0x26, 0x01, 0x45, 0x82, 0x50, 0x2c, 0xf1, + 0x0d, 0x29, 0x38, 0xf6, 0x36, 0xb2, 0x9a, 0xb3, 0x8d, 0x5f, 0x12, 0x85, 0xbf, 0xc0, 0x27, 0x7e, + 0x06, 0xbf, 0x81, 0xcf, 0xfc, 0x2b, 0x3e, 0x30, 0xb3, 0x2f, 0xbe, 0x5d, 0xf7, 0x42, 0x44, 0xd5, + 0x4f, 0xb7, 0x33, 0x3b, 0x9e, 0x99, 0x7d, 0xf6, 0x99, 0x9d, 0x39, 0x80, 0xa4, 0xa8, 0xc4, 0x4e, + 0x59, 0x15, 0x4d, 0xc1, 0x06, 0xe5, 0x29, 0x1f, 0x43, 0x70, 0xb8, 0x28, 0x9b, 0x2b, 0xfe, 0x08, + 0xfc, 0xe7, 0x45, 0xca, 0x18, 0x0c, 0xf3, 0x78, 0x21, 0xe6, 0xde, 0x7b, 0xde, 0xc3, 0x69, 0x24, + 0xd7, 0xa4, 0x4b, 0x45, 0x9d, 0xcc, 0x07, 0x4a, 0x47, 0x6b, 0xbe, 0x0d, 0x43, 0x34, 0xaf, 0xd9, + 0x7d, 0x18, 0x96, 0xf8, 0x8b, 0xf6, 0xfe, 0xc3, 0xd9, 0xee, 0x78, 0xa7, 0x3c, 0xdd, 0x41, 0x7d, + 0x24, 0x95, 0xfc, 0x6f, 0x0f, 0x86, 0xc7, 0x45, 0x2a, 0x56, 0x7a, 0x0d, 0x61, 0x22, 0xf2, 0xb4, + 0x2c, 0xb2, 0xbc, 0xd1, 0x9e, 0x3b, 0x99, 0xcd, 0x61, 0x8c, 0x0e, 0xe4, 0x27, 0xbe, 0xdc, 0x32, + 0x22, 0xdb, 0x82, 0x51, 0xd9, 0x9e, 0x9e, 0x67, 0xc9, 0x7c, 0x88, 0x1b, 0x93, 0x48, 0x4b, 0x6c, + 0x1b, 0xfc, 0xa4, 0x6c, 0xe7, 0x81, 0x4c, 0xe3, 0x0e, 0xa5, 0x41, 0x81, 0x77, 0xf6, 0xcb, 0xf6, + 0x30, 0x6f, 0xaa, 0xab, 0x88, 0x76, 0xc3, 0xcf, 0x60, 0x62, 0x14, 0x6c, 0x13, 0xfc, 0x97, 0xe2, + 0x4a, 0x67, 0x44, 0x4b, 0x76, 0x17, 0x82, 0x8b, 0xf8, 0xbc, 0x15, 0x32, 0x1b, 0x3f, 0x52, 0xc2, + 0x97, 0x83, 0x2f, 0x3c, 0xfe, 0x21, 0x04, 0xe4, 0xad, 0x66, 0x0f, 0x20, 0xc8, 0x69, 0xa1, 0x8f, + 0x3b, 0x31, 0x71, 0x22, 0xa5, 0xe6, 0x57, 0x30, 0xdd, 0x2f, 0xf2, 0x26, 0xce, 0x72, 0x51, 0xb1, + 0x75, 0x18, 0x64, 0xa9, 0x0e, 0x80, 0x2b, 0xfb, 0x50, 0x03, 0xf7, 0x50, 0x08, 0x05, 0x7d, 0x6f, + 0x9d, 0xb7, 0x93, 0x3b, 0xe8, 0x86, 0xee, 0x85, 0x64, 0xf9, 0x8b, 0x02, 0x4f, 0x2b, 0x75, 0xb4, + 0xe6, 0x7b, 0x00, 0x5d, 0xe8, 0x9a, 0x3d, 0xa2, 0x8b, 0x36, 0x92, 0xce, 0x76, 0x8d, 0xb2, 0xed, + 0x6c, 0x22, 0xcb, 0x80, 0xbf, 0x03, 0xb3, 0x6e, 0xe3, 0xe8, 0xa0, 0x9f, 0x39, 0xff, 0x04, 0x6e, + 0x5b, 0xdb, 0x35, 0x7b, 0x1f, 0xfc, 0xac, 0xbb, 0xf3, 0x0d, 0xc7, 0xed, 0xd1, 0x41, 0x44, 0x7b, + 0xfc, 0x73, 0x58, 0x7b, 0x9a, 0xa6, 0x48, 0x85, 0x1f, 0xca, 0x26, 0x2b, 0xf2, 0xfa, 0x7f, 0x10, + 0x6b, 0xed, 0x5b, 0xd1, 0xfc, 0xf7, 0x87, 0xfc, 0x37, 0x58, 0x47, 0xef, 0x84, 0xbc, 0xb1, 0xb2, + 0x21, 0xf4, 0x7a, 0x10, 0xbe, 0x51, 0xa6, 0xf1, 0x6f, 0x60, 0x1d, 0x13, 0xb4, 0x63, 0x5b, 0x3e, + 0xbc, 0xeb, 0x2f, 0x76, 0xe0, 0x66, 0xc5, 0x3f, 0x86, 0xcd, 0xef, 0xb2, 0x5a, 0x3a, 0xaa, 0x6f, + 0xf4, 0xc4, 0x0b, 0xb8, 0xf3, 0xac, 0xcd, 0xce, 0xd3, 0xa3, 0x45, 0x7c, 0x26, 0x2c, 0x68, 0x2a, + 0x51, 0x16, 0x06, 0x1a, 0x5a, 0x93, 0x8b, 0x0b, 0xbc, 0x52, 0xdc, 0x37, 0x2c, 0xd3, 0x22, 0x31, + 0xbe, 0xc5, 0x6b, 0x55, 0xc7, 0xa4, 0x25, 0xa5, 0x17, 0x57, 0x4d, 0xf6, 0x22, 0x4e, 0x1a, 0xcd, + 0xaf, 0x4e, 0xe6, 0xff, 0x0c, 0x60, 0xed, 0x40, 0x94, 0xe7, 0xc5, 0x95, 0x89, 0x86, 0xf5, 0x51, + 0x97, 0x22, 0xa9, 0x75, 0x38, 0x25, 0x50, 0xbc, 0xb8, 0x2c, 0x6d, 0x56, 0x6b, 0x91, 0xec, 0x33, + 0xca, 0x56, 0x47, 0x54, 0x82, 0x7d, 0xc4, 0xe1, 0xf5, 0x60, 0x05, 0xbd, 0x2b, 0x7c, 0x00, 0x20, + 0xa8, 0x6c, 0xd5, 0x25, 0x8e, 0xe4, 0xae, 0xa5, 0xc1, 0x67, 0x68, 0x8a, 0x05, 0x7e, 0xf2, 0x6b, + 0x5b, 0x34, 0xf1, 0x7c, 0x8c, 0xdb, 0x5e, 0x34, 0x41, 0xc5, 0x8f, 0x24, 0x53, 0x22, 0x49, 0xd1, + 0xe2, 0x77, 0x13, 0xdc, 0x08, 0x22, 0x25, 0x10, 0x1c, 0x22, 0xbf, 0x98, 0x4f, 0x91, 0xc4, 0x08, + 0x07, 0x2e, 0xd9, 0x1e, 0x26, 0x20, 0x9a, 0xcb, 0xa2, 0x7a, 0x59, 0xcf, 0x41, 0x72, 0xfb, 0x5d, + 0xe2, 0xb6, 0x83, 0xc2, 0xce, 0xb1, 0xb6, 0x50, 0xcf, 0x4a, 0xf7, 0x01, 0xb9, 0xab, 0xe2, 0xcb, + 0xf9, 0x4c, 0x72, 0x85, 0x96, 0xe1, 0x1e, 0xac, 0x39, 0xc6, 0x37, 0x3d, 0x39, 0x53, 0xfb, 0xc9, + 0x39, 0x82, 0xf5, 0x9f, 0xca, 0xb3, 0x2a, 0x5e, 0xb2, 0xec, 0xe6, 0xa2, 0x5b, 0x22, 0x3e, 0xb0, + 0x10, 0xe7, 0xa7, 0xc0, 0x22, 0xb1, 0x28, 0x2e, 0x84, 0xc3, 0x9d, 0xd7, 0x22, 0x2d, 0x15, 0x85, + 0x74, 0x5a, 0xe3, 0xa5, 0x12, 0x6e, 0x5a, 0xc2, 0xd7, 0x67, 0x76, 0x58, 0x55, 0x45, 0x75, 0x20, + 0x30, 0xa3, 0x73, 0x22, 0x66, 0x82, 0x9f, 0x48, 0xcf, 0x7e, 0x24, 0xd7, 0x14, 0x70, 0x21, 0xea, + 0x7a, 0x99, 0x9e, 0x11, 0xf9, 0x9f, 0x9e, 0x4d, 0xee, 0xef, 0x95, 0x96, 0x42, 0xd5, 0x4d, 0xdc, + 0xb4, 0x86, 0x6f, 0x5a, 0xa2, 0xf4, 0xb0, 0x7d, 0x9d, 0x55, 0x68, 0x66, 0xd2, 0x33, 0x32, 0x01, + 0x20, 0x28, 0x0d, 0x43, 0x39, 0x29, 0x28, 0x4f, 0x95, 0x88, 0x17, 0x9a, 0x71, 0x5a, 0x62, 0xbb, + 0x70, 0x5b, 0x1a, 0x9c, 0xa4, 0x32, 0x6b, 0x49, 0x3a, 0x0d, 0xad, 0x75, 0x98, 0x68, 0x26, 0x96, + 0x02, 0xff, 0x63, 0x00, 0x5b, 0xfb, 0xf8, 0x75, 0x23, 0x3a, 0xf4, 0x4d, 0xc2, 0xaf, 0x87, 0xa8, + 0x7a, 0x6b, 0xfd, 0xae, 0x4b, 0xac, 0x7a, 0xef, 0xbb, 0x63, 0x05, 0xf6, 0xb1, 0x30, 0x5e, 0xdd, + 0x26, 0x09, 0xe1, 0x30, 0x92, 0xac, 0x33, 0x22, 0xfb, 0x54, 0x35, 0xc3, 0xb1, 0xa4, 0xca, 0xb6, + 0xa4, 0xca, 0xca, 0x94, 0xdf, 0x50, 0x7b, 0xfc, 0xc5, 0x21, 0x98, 0x81, 0xa3, 0x23, 0xa3, 0xd7, + 0x2b, 0x7f, 0x93, 0xf4, 0xc0, 0x4d, 0x1a, 0x41, 0xd2, 0x84, 0x30, 0xe4, 0xea, 0x64, 0xfe, 0x33, + 0x6c, 0xa9, 0x08, 0xaf, 0x80, 0xbe, 0xa2, 0xc9, 0x5e, 0xe3, 0xdf, 0xe2, 0x9f, 0xef, 0xf2, 0xef, + 0x77, 0x0f, 0xde, 0xd2, 0xc5, 0x76, 0xa3, 0xff, 0x7b, 0x30, 0xca, 0xc5, 0xe5, 0x09, 0xea, 0x74, + 0x8d, 0xa1, 0x74, 0x94, 0xb2, 0xb7, 0xe9, 0xe9, 0xb8, 0x3c, 0xb1, 0xfb, 0x08, 0xca, 0xc7, 0xce, + 0xe5, 0x0d, 0xaf, 0xb9, 0xbc, 0xc0, 0xc9, 0x73, 0xf7, 0xaf, 0x00, 0xc6, 0xfb, 0x38, 0xa4, 0x45, + 0xcf, 0xf7, 0xb1, 0xe6, 0x27, 0xd4, 0x23, 0xe4, 0xa4, 0x35, 0x95, 0xbc, 0xa4, 0x59, 0x2d, 0x9c, + 0xe8, 0x31, 0xab, 0xe6, 0xb7, 0xd8, 0x07, 0x30, 0x52, 0x8d, 0x96, 0xc9, 0xa9, 0xc7, 0x69, 0xba, + 0xa1, 0x99, 0xc7, 0x94, 0x9d, 0xea, 0xab, 0xca, 0xce, 0xe9, 0xb1, 0xb6, 0xdd, 0x47, 0x30, 0xd6, + 0xad, 0x95, 0x31, 0xed, 0xd0, 0xea, 0x75, 0x61, 0x37, 0xf2, 0x28, 0x53, 0xdd, 0x09, 0x95, 0xa9, + 0xdb, 0x16, 0x1d, 0x53, 0x9c, 0x20, 0xf4, 0x41, 0xd4, 0x20, 0x75, 0x97, 0xf6, 0xfa, 0xed, 0x2f, + 0x9c, 0x9a, 0x2f, 0xe8, 0x60, 0x8f, 0xe1, 0x36, 0x3a, 0x5c, 0x8e, 0x53, 0xfd, 0x27, 0x2f, 0x74, + 0xe7, 0x19, 0xfc, 0xe2, 0x89, 0x1c, 0x1d, 0xac, 0x29, 0x68, 0xb3, 0xf7, 0x49, 0x1d, 0xae, 0x3b, + 0x1a, 0x0a, 0xf3, 0x15, 0xc0, 0xf2, 0xed, 0x61, 0xf7, 0x68, 0xff, 0x95, 0x46, 0x1b, 0xf6, 0xd4, + 0x9a, 0x1c, 0xfc, 0xd6, 0x63, 0x8f, 0x3d, 0x83, 0x8d, 0x5e, 0x71, 0x29, 0x80, 0x9d, 0xae, 0x11, + 0x86, 0xd7, 0x17, 0x21, 0xfa, 0x38, 0x84, 0xcd, 0x3e, 0xff, 0x14, 0xa2, 0x6e, 0x0b, 0x08, 0xef, + 0x5b, 0xba, 0x15, 0x6e, 0x9e, 0xc2, 0x46, 0xaf, 0x4a, 0x56, 0x20, 0x20, 0x33, 0x59, 0x5d, 0x4c, + 0xe8, 0xe2, 0x6b, 0x98, 0x59, 0xa5, 0xcc, 0xb6, 0x96, 0xc6, 0x0e, 0x1e, 0x7d, 0x7d, 0xe7, 0xe0, + 0x74, 0x24, 0xff, 0x5a, 0x3c, 0xf9, 0x37, 0x00, 0x00, 0xff, 0xff, 0xac, 0x37, 0x89, 0xa9, 0x68, + 0x0c, 0x00, 0x00, } diff --git a/rpc/gen/core.proto b/rpc/gen/core.proto index 83ec20310..d33e8b45b 100644 --- a/rpc/gen/core.proto +++ b/rpc/gen/core.proto @@ -95,6 +95,7 @@ message BuildImageOptions { string repo = 1; string version = 2; string uid = 3; + string artifact = 4; } message DeployOptions { diff --git a/rpc/rpc.go b/rpc/rpc.go index 184a608d2..a8847dc8e 100644 --- a/rpc/rpc.go +++ b/rpc/rpc.go @@ -144,7 +144,7 @@ func (v *Virbranium) GetContainers(ctx context.Context, cids *pb.ContainerIDs) ( // streamed returned functions // caller must ensure that timeout will not be too short because these actions take a little time func (v *Virbranium) BuildImage(opts *pb.BuildImageOptions, stream pb.CoreRPC_BuildImageServer) error { - ch, err := v.cluster.BuildImage(opts.Repo, opts.Version, opts.Uid) + ch, err := v.cluster.BuildImage(opts.Repo, opts.Version, opts.Uid, opts.Artifact) if err != nil { return err } diff --git a/scheduler/simple/magnesium.go b/scheduler/simple/magnesium.go index 9313cc519..7576d7ec1 100644 --- a/scheduler/simple/magnesium.go +++ b/scheduler/simple/magnesium.go @@ -7,14 +7,14 @@ import ( "gitlab.ricebook.net/platform/core/types" ) -type Magnesium struct { +type magnesium struct { sync.Mutex } // Get a random node. // Actually it's not random, it's the one with biggest cpu quota. // Returns the node name. -func (m *Magnesium) RandomNode(nodes map[string]types.CPUMap) (string, error) { +func (m *magnesium) RandomNode(nodes map[string]types.CPUMap) (string, error) { m.Lock() defer m.Unlock() @@ -40,7 +40,7 @@ func (m *Magnesium) RandomNode(nodes map[string]types.CPUMap) (string, error) { // Select nodes for deploying. // Use round robin method to select, in order to make scheduler average. // TODO Outside this method, caller should update corresponding nodes with `nodes` as their CPU, which is weird -func (m *Magnesium) SelectNodes(nodes map[string]types.CPUMap, quota int, num int) (map[string][]types.CPUMap, error) { +func (m *magnesium) SelectNodes(nodes map[string]types.CPUMap, quota int, num int) (map[string][]types.CPUMap, error) { m.Lock() defer m.Unlock() @@ -91,3 +91,7 @@ func totalQuota(nodes map[string]types.CPUMap) int { } return value } + +func New() *magnesium { + return &magnesium{} +} diff --git a/scheduler/simple/magnesium_test.go b/scheduler/simple/magnesium_test.go index 9aced2694..fef9daa05 100644 --- a/scheduler/simple/magnesium_test.go +++ b/scheduler/simple/magnesium_test.go @@ -8,7 +8,7 @@ import ( ) func TestRandomNode(t *testing.T) { - m := &Magnesium{} + m := &magnesium{} _, err := m.RandomNode(map[string]types.CPUMap{}) assert.Error(t, err) assert.Equal(t, err.Error(), "No nodes provide to choose one") @@ -41,7 +41,7 @@ func TestRandomNode(t *testing.T) { } func TestSelectNodes(t *testing.T) { - m := &Magnesium{} + m := &magnesium{} _, err := m.SelectNodes(map[string]types.CPUMap{}, 1, 1) assert.Error(t, err) assert.Equal(t, err.Error(), "No nodes provide to choose some") @@ -133,7 +133,7 @@ func TestTotalQuota(t *testing.T) { } func TestSelectPublicNodes(t *testing.T) { - m := &Magnesium{} + m := &magnesium{} _, err := m.SelectNodes(map[string]types.CPUMap{}, 1, 1) assert.Error(t, err) assert.Equal(t, err.Error(), "No nodes provide to choose some") diff --git a/source/gitlab/cesium.go b/source/gitlab/cesium.go new file mode 100644 index 000000000..7d0eb645f --- /dev/null +++ b/source/gitlab/cesium.go @@ -0,0 +1,157 @@ +package gitlab + +import ( + "archive/zip" + "bytes" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "strings" + + log "github.com/Sirupsen/logrus" + "gitlab.ricebook.net/platform/core/types" + "gopkg.in/libgit2/git2go.v23" +) + +type cesium struct { + http.Client + config types.Config +} + +const gitUser = "git" + +func certificateCheckCallback(cert *git.Certificate, valid bool, hostname string) git.ErrorCode { + return git.ErrorCode(0) +} + +// clone code from repository into path, by revision +func (c *cesium) SourceCode(repository, path, revision string) error { + pubkey := c.config.Git.PublicKey + prikey := c.config.Git.PrivateKey + + if !strings.HasPrefix(repository, "git@") { + return fmt.Errorf("Only support ssh protocol(%q), use git@...", repository) + } + if _, err := os.Stat(pubkey); os.IsNotExist(err) { + return fmt.Errorf("Public Key not found(%q)", pubkey) + } + if _, err := os.Stat(prikey); os.IsNotExist(err) { + return fmt.Errorf("Private Key not found(%q)", prikey) + } + + credentialsCallback := func(url, username string, allowedTypes git.CredType) (git.ErrorCode, *git.Cred) { + ret, cred := git.NewCredSshKey(gitUser, pubkey, prikey, "") + return git.ErrorCode(ret), &cred + } + + cloneOpts := &git.CloneOptions{ + FetchOptions: &git.FetchOptions{ + RemoteCallbacks: git.RemoteCallbacks{ + CredentialsCallback: credentialsCallback, + CertificateCheckCallback: certificateCheckCallback, + }, + }, + } + + repo, err := git.Clone(repository, path, cloneOpts) + if err != nil { + return err + } + + if err := repo.CheckoutHead(nil); err != nil { + return err + } + + object, err := repo.RevparseSingle(revision) + if err != nil { + return err + } + defer object.Free() + + // below is code for v24 + // which is fucking unstable + // ------------------------- + // + // tree, err := object.AsTree() + // if err != nil { + // return err + // } + obj, err := object.Peel(git.ObjectTree) + if err != nil { + return err + } + + tree, ok := obj.(*git.Tree) + if !ok { + return fmt.Errorf("git object is not a tree") + } + return repo.CheckoutTree(tree, &git.CheckoutOpts{Strategy: git.CheckoutSafe}) +} + +// Download artifact to path, and unzip it +// artifact is like "http://gitlab.ricebook.net/api/v3/projects/:project_id/builds/:build_id/artifacts" +func (c *cesium) Artifact(artifact, path string) error { + req, err := http.NewRequest("GET", artifact, nil) + if err != nil { + return err + } + + req.Header.Add("PRIVATE-TOKEN", c.config.Git.GitlabToken) + resp, err := c.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return fmt.Errorf("Download artifact error %q", artifact) + } + + log.Debugf("Downloading artifacts from %q", artifact) + content, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + log.Debugf("Extracting files from %q", artifact) + reader, err := zip.NewReader(bytes.NewReader(content), int64(len(content))) + if err != nil { + return err + } + + // extract files from zipfile + for _, f := range reader.File { + zipped, err := f.Open() + if err != nil { + return err + } + + defer zipped.Close() + + p := filepath.Join(path, f.Name) + + if f.FileInfo().IsDir() { + os.MkdirAll(p, f.Mode()) + continue + } + + writer, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE, f.Mode()) + if err != nil { + return err + } + + defer writer.Close() + if _, err = io.Copy(writer, zipped); err != nil { + return err + } + } + log.Debugf("Extraction from %q done", artifact) + return nil +} + +func New(config types.Config) *cesium { + return &cesium{config: config} +} diff --git a/source/source.go b/source/source.go new file mode 100644 index 000000000..498cc049f --- /dev/null +++ b/source/source.go @@ -0,0 +1,8 @@ +package source + +type Source interface { + // Get source code from repository into path by revision + SourceCode(repository, path, revision string) error + // Get related artifact by artifact into path + Artifact(artifact, path string) error +} diff --git a/types/config.go b/types/config.go index d87260253..71d3f1345 100644 --- a/types/config.go +++ b/types/config.go @@ -11,8 +11,9 @@ type Config struct { } type GitConfig struct { - PublicKey string `yaml:"public_key"` - PrivateKey string `yaml:"private_key"` + PublicKey string `yaml:"public_key"` + PrivateKey string `yaml:"private_key"` + GitlabToken string `yaml:"gitlab_token"` } type DockerConfig struct {