add artifacts retriever

cluster/calcium/build_image.go

@@ -11,7 +11,6 @@ import (
 
 	"github.com/docker/docker/pkg/archive"
 	enginetypes "github.com/docker/engine-api/types"
-	"gitlab.ricebook.net/platform/core/git"
 	"gitlab.ricebook.net/platform/core/types"
 	"gitlab.ricebook.net/platform/core/utils"
 	"golang.org/x/net/context"
@@ -87,7 +86,7 @@ func getRandomNode(c *Calcium, podname string) (*types.Node, error) {
 //             ├─ Dockerfile
 //             ├─ launcher
 //             ├─ launcheroot
-func (c *Calcium) BuildImage(repository, version, uid string) (chan *types.BuildImageMessage, error) {
+func (c *Calcium) BuildImage(repository, version, uid, artifact string) (chan *types.BuildImageMessage, error) {
 	ch := make(chan *types.BuildImageMessage)
 
 	// use pod `dev` to build image
@@ -112,10 +111,18 @@ func (c *Calcium) BuildImage(repository, version, uid string) (chan *types.Build
 	// clone code into cloneDir
 	// which is under buildDir and named as repository name
 	cloneDir := filepath.Join(buildDir, reponame)
-	if err := git.CloneRepository(repository, cloneDir, version, c.config.Git.PublicKey, c.config.Git.PrivateKey); err != nil {
+	if err := c.source.SourceCode(repository, cloneDir, version); err != nil {
 		return ch, err
 	}
 
+	// get artifact into cloneDir, only when artifact is not empty
+	// TODO need to separate from code or just like this?
+	if artifact != "" {
+		if err := c.source.Artifact(artifact, cloneDir); err != nil {
+			return ch, err
+		}
+	}
+
 	// ensure .git directory is removed
 	// we don't want any history files to be retrieved
 	if err := os.RemoveAll(filepath.Join(cloneDir, ".git")); err != nil {

cluster/calcium/build_image_test.go

@@ -6,13 +6,15 @@ import (
 	"github.com/stretchr/testify/assert"
 	"gitlab.ricebook.net/platform/core/network/calico"
 	"gitlab.ricebook.net/platform/core/scheduler/simple"
+	"gitlab.ricebook.net/platform/core/source/gitlab"
 	"gitlab.ricebook.net/platform/core/store/mock"
 	"gitlab.ricebook.net/platform/core/types"
 )
 
 func TestGetRandomNode(t *testing.T) {
 	store := &mockstore.MockStore{}
-	c := &Calcium{store: store, config: types.Config{}, scheduler: &simplescheduler.Magnesium{}, network: &calico.Titanium{}}
+	config := types.Config{}
+	c := &Calcium{store: store, config: config, scheduler: simplescheduler.New(), network: calico.New(), source: gitlab.New(config)}
 
 	n1 := &types.Node{Name: "node1", Podname: "podname", Endpoint: "tcp://10.0.0.1:2376", CPU: types.CPUMap{"0": 10, "1": 10}}
 	n2 := &types.Node{Name: "node2", Podname: "podname", Endpoint: "tcp://10.0.0.2:2376", CPU: types.CPUMap{"0": 10, "1": 10}}

cluster/calcium/cluster.go

@@ -7,6 +7,8 @@ import (
 	"gitlab.ricebook.net/platform/core/network/calico"
 	"gitlab.ricebook.net/platform/core/scheduler"
 	"gitlab.ricebook.net/platform/core/scheduler/simple"
+	"gitlab.ricebook.net/platform/core/source"
+	"gitlab.ricebook.net/platform/core/source/gitlab"
 	"gitlab.ricebook.net/platform/core/store"
 	"gitlab.ricebook.net/platform/core/store/etcd"
 	"gitlab.ricebook.net/platform/core/types"
@@ -18,6 +20,7 @@ type Calcium struct {
 	config    types.Config
 	scheduler scheduler.Scheduler
 	network   network.Network
+	source    source.Source
 }
 
 func New(config types.Config) (*Calcium, error) {
@@ -26,8 +29,9 @@ func New(config types.Config) (*Calcium, error) {
 		return nil, err
 	}
 
-	scheduler := &simplescheduler.Magnesium{}
-	titanium := &calico.Titanium{}
+	scheduler := simplescheduler.New()
+	titanium := calico.New()
+	source := gitlab.New(config)
 
-	return &Calcium{store: store, config: config, scheduler: scheduler, network: titanium}, nil
+	return &Calcium{store: store, config: config, scheduler: scheduler, network: titanium, source: source}, nil
 }

cluster/calcium/meta_test.go

@@ -7,13 +7,15 @@ import (
 	"github.com/stretchr/testify/assert"
 	"gitlab.ricebook.net/platform/core/network/calico"
 	"gitlab.ricebook.net/platform/core/scheduler/simple"
+	"gitlab.ricebook.net/platform/core/source/gitlab"
 	"gitlab.ricebook.net/platform/core/store/mock"
 	"gitlab.ricebook.net/platform/core/types"
 )
 
 func TestListPods(t *testing.T) {
 	store := &mockstore.MockStore{}
-	c := &Calcium{store: store, config: types.Config{}, scheduler: &simplescheduler.Magnesium{}, network: &calico.Titanium{}}
+	config := types.Config{}
+	c := &Calcium{store: store, config: config, scheduler: simplescheduler.New(), network: calico.New(), source: gitlab.New(config)}
 
 	store.On("GetAllPods").Return([]*types.Pod{
 		&types.Pod{Name: "pod1", Desc: "desc1"},
@@ -33,7 +35,8 @@ func TestListPods(t *testing.T) {
 
 func TestAddPod(t *testing.T) {
 	store := &mockstore.MockStore{}
-	c := &Calcium{store: store, config: types.Config{}, scheduler: &simplescheduler.Magnesium{}, network: &calico.Titanium{}}
+	config := types.Config{}
+	c := &Calcium{store: store, config: config, scheduler: simplescheduler.New(), network: calico.New(), source: gitlab.New(config)}
 
 	store.On("AddPod", "pod1", "desc1").Return(&types.Pod{Name: "pod1", Desc: "desc1"}, nil)
 	store.On("AddPod", "pod2", "desc2").Return(nil, fmt.Errorf("Etcd Error"))
@@ -50,7 +53,8 @@ func TestAddPod(t *testing.T) {
 
 func TestGetPods(t *testing.T) {
 	store := &mockstore.MockStore{}
-	c := &Calcium{store: store, config: types.Config{}, scheduler: &simplescheduler.Magnesium{}, network: &calico.Titanium{}}
+	config := types.Config{}
+	c := &Calcium{store: store, config: config, scheduler: simplescheduler.New(), network: calico.New(), source: gitlab.New(config)}
 
 	store.On("GetPod", "pod1").Return(&types.Pod{Name: "pod1", Desc: "desc1"}, nil).Once()
 	store.On("GetPod", "pod2").Return(nil, fmt.Errorf("Not found")).Once()

cluster/cluster.go

@@ -16,7 +16,7 @@ type Cluster interface {
 	GetContainers(ids []string) ([]*types.Container, error)
 
 	// cluster methods
-	BuildImage(repository, version, uid string) (chan *types.BuildImageMessage, error)
+	BuildImage(repository, version, uid, artifact string) (chan *types.BuildImageMessage, error)
 	CreateContainer(specs types.Specs, opts *types.DeployOptions) (chan *types.CreateContainerMessage, error)
 	UpgradeContainer(ids []string, image string) (chan *types.UpgradeContainerMessage, error)
 	RemoveContainer(ids []string) (chan *types.RemoveContainerMessage, error)

devtools/client.py

@@ -137,10 +137,12 @@ def add_node(ctx, nodename, endpoint, podname, public):
 @click.argument('repo')
 @click.argument('version')
 @click.argument('uid')
+@click.option('artifact', default='')
 @click.pass_context
-def build_image(ctx, repo, version, uid):
+def build_image(ctx, repo, version, uid, artifact):
+    # artifact = 'http://gitlab.ricebook.net/api/v3/projects/245/builds/1815/artifacts'
     stub = _get_stub(ctx)
-    opts = pb.BuildImageOptions(repo=repo, version=version, uid=uid)
+    opts = pb.BuildImageOptions(repo=repo, version=version, uid=uid, artifact=artifact)
 
     try:
         for m in stub.BuildImage(opts, 3600):