From 1572cdd9bee5b1c9c17e8907993ae9fb23a5e52d Mon Sep 17 00:00:00 2001 From: gongweibao Date: Tue, 6 Feb 2018 07:58:14 +0000 Subject: [PATCH 1/7] test --- .travis.yml | 2 +- k8s/fluid_raw_job/pserver.yaml | 43 ++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 k8s/fluid_raw_job/pserver.yaml diff --git a/.travis.yml b/.travis.yml index 79e8b1d2..c3238429 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ matrix: - cd go && glide install && go get k8s.io/kubernetes || echo 1 - bash ./vendor/k8s.io/code-generator/generate-groups.sh "deepcopy,client,informer,lister" github.com/PaddlePaddle/cloud/go/pkg/client github.com/PaddlePaddle/cloud/go/pkg/apis paddlepaddle:v1alpha1 - grep "github.com/paddlepaddle/cloud" -nR pkg/client | awk -F ':' '{print $1}' | xargs sed -i 's|github.com/paddlepaddle/cloud|github.com/PaddlePaddle/cloud|g' - - bash .tools/gen_config.sh && glide install --strip-vendor && go test $(glide novendor) + - glide install --strip-vendor && go test $(glide novendor) - language: python python: 2.7 sudo: required diff --git a/k8s/fluid_raw_job/pserver.yaml b/k8s/fluid_raw_job/pserver.yaml new file mode 100644 index 00000000..35b43262 --- /dev/null +++ b/k8s/fluid_raw_job/pserver.yaml @@ -0,0 +1,43 @@ +apiVersion: extensions/v1beta1 +kind: ReplicaSet +metadata: + name: rawjob-pserver +spec: + replicas: 4 + template: + metadata: + labels: + paddle-job-pserver: fluid_raw_job + spec: + hostNetwork: true + imagePullSecrets: + - name: job-registry-secret + containers: + - name: pserver + image: "yourimage" + imagePullPolicy: Always + ports: + - name: jobport-30487 + containerPort: 30487 + env: + - name: PADDLE_JOB_NAME + value: rawjob + - name: TRAINERS + value: "4" + - name: PSERVERS + value: "4" + - name: LD_LIBRARY_PATH + value: "/usr/local/nvidia/lib64" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: "metadata.namespace" + command: ["python", "trainer.py"] + resources: + requests: + memory: 8Gi + cpu: 1 + limits: + memory: 12Gi + cpu: 1 + From 289e68163eebce30207213f3399f8bbeb919a2f3 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Tue, 6 Feb 2018 08:02:53 +0000 Subject: [PATCH 2/7] clean up --- k8s/fluid_raw_job/pserver.yaml | 43 ---------------------------------- 1 file changed, 43 deletions(-) delete mode 100644 k8s/fluid_raw_job/pserver.yaml diff --git a/k8s/fluid_raw_job/pserver.yaml b/k8s/fluid_raw_job/pserver.yaml deleted file mode 100644 index 35b43262..00000000 --- a/k8s/fluid_raw_job/pserver.yaml +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: ReplicaSet -metadata: - name: rawjob-pserver -spec: - replicas: 4 - template: - metadata: - labels: - paddle-job-pserver: fluid_raw_job - spec: - hostNetwork: true - imagePullSecrets: - - name: job-registry-secret - containers: - - name: pserver - image: "yourimage" - imagePullPolicy: Always - ports: - - name: jobport-30487 - containerPort: 30487 - env: - - name: PADDLE_JOB_NAME - value: rawjob - - name: TRAINERS - value: "4" - - name: PSERVERS - value: "4" - - name: LD_LIBRARY_PATH - value: "/usr/local/nvidia/lib64" - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: "metadata.namespace" - command: ["python", "trainer.py"] - resources: - requests: - memory: 8Gi - cpu: 1 - limits: - memory: 12Gi - cpu: 1 - From 463dffe4ad44ba1cf26dfb2c503ec2bb1e1a2b02 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Tue, 6 Feb 2018 08:34:37 +0000 Subject: [PATCH 3/7] clean up --- go/.tools/gen_config.sh | 10 ---------- go/Dockerfile | 4 ---- go/cmd/paddlecloud/paddlecloud.go | 4 ++++ go/cmd/paddlectl/main.go | 5 +++++ go/filemanager/pfsmodules/upload.go | 2 +- go/paddlecloud/submit.go | 2 +- go/paddlectl/submit.go | 2 +- 7 files changed, 12 insertions(+), 17 deletions(-) delete mode 100644 go/.tools/gen_config.sh diff --git a/go/.tools/gen_config.sh b/go/.tools/gen_config.sh deleted file mode 100644 index f42d951b..00000000 --- a/go/.tools/gen_config.sh +++ /dev/null @@ -1,10 +0,0 @@ -#/bin/bash -mkdir -p ~/.paddle -cat > ~/.paddle/config << EOF -datacenters: -- name: datacenter1 - username: your-user-name - password: your-secret - endpoint: http://127.0.0.1:8080 -current-datacenter: datacenter1 -EOF diff --git a/go/Dockerfile b/go/Dockerfile index 6e959fa4..a12b202a 100644 --- a/go/Dockerfile +++ b/go/Dockerfile @@ -1,7 +1,3 @@ FROM ubuntu:16.04 - -ADD .tools /pfsserver/.tools -RUN bash /pfsserver/.tools/gen_config.sh - ADD ./cmd/pfsserver/pfsserver /pfsserver/ RUN mkdir /pfsserver/log diff --git a/go/cmd/paddlecloud/paddlecloud.go b/go/cmd/paddlecloud/paddlecloud.go index a2a5872b..c77d8bac 100644 --- a/go/cmd/paddlecloud/paddlecloud.go +++ b/go/cmd/paddlecloud/paddlecloud.go @@ -7,10 +7,14 @@ import ( pfsmod "github.com/PaddlePaddle/cloud/go/filemanager/pfsmodules" "github.com/PaddlePaddle/cloud/go/paddlecloud" + "github.com/PaddlePaddle/cloud/go/utils/config" "github.com/google/subcommands" ) func main() { + pfsmod.Config = config.ParseDefaultConfig() + paddlecloud.Config = config.ParseDefaultConfig() + subcommands.Register(subcommands.HelpCommand(), "") subcommands.Register(subcommands.CommandsCommand(), "") subcommands.Register(&paddlecloud.SubmitCmd{}, "") diff --git a/go/cmd/paddlectl/main.go b/go/cmd/paddlectl/main.go index 5bbbe0d2..90844e08 100644 --- a/go/cmd/paddlectl/main.go +++ b/go/cmd/paddlectl/main.go @@ -5,11 +5,16 @@ import ( "flag" "os" + "github.com/PaddlePaddle/cloud/go/paddlecloud" "github.com/PaddlePaddle/cloud/go/paddlectl" + "github.com/PaddlePaddle/cloud/go/utils/config" "github.com/google/subcommands" ) func main() { + pfsmod.Config = config.ParseDefaultConfig() + paddlecloud.Config = config.ParseDefaultConfig() + subcommands.Register(subcommands.HelpCommand(), "") subcommands.Register(subcommands.CommandsCommand(), "") subcommands.Register(&paddlectl.SubmitCmd{}, "") diff --git a/go/filemanager/pfsmodules/upload.go b/go/filemanager/pfsmodules/upload.go index 8da81b91..b58ebe27 100644 --- a/go/filemanager/pfsmodules/upload.go +++ b/go/filemanager/pfsmodules/upload.go @@ -15,7 +15,7 @@ import ( ) // Config is global config object for pfs commandline -var Config = config.ParseDefaultConfig() +var Config *config.SubmitConfig = nil type uploadChunkResponse struct { Err string `json:"err"` diff --git a/go/paddlecloud/submit.go b/go/paddlecloud/submit.go index 18d4680f..6f6bdc25 100644 --- a/go/paddlecloud/submit.go +++ b/go/paddlecloud/submit.go @@ -21,7 +21,7 @@ const ( ) // Config is global config object for paddlecloud commandline -var Config = config.ParseDefaultConfig() +var Config *config.SubmitConfig = nil // SubmitCmd define the subcommand of submitting paddle training jobs. type SubmitCmd struct { diff --git a/go/paddlectl/submit.go b/go/paddlectl/submit.go index 00f6d4a6..c7041239 100644 --- a/go/paddlectl/submit.go +++ b/go/paddlectl/submit.go @@ -22,7 +22,7 @@ const ( ) // Config is global config object for paddlectl commandline -var Config = config.ParseDefaultConfig() +var Config *config.SumitConfig = nil // SubmitCmd define the subcommand of submitting paddle training jobs. type SubmitCmd struct { From 556809b84e192237201ee1c2f085215469f94c1d Mon Sep 17 00:00:00 2001 From: gongweibao Date: Tue, 6 Feb 2018 08:36:45 +0000 Subject: [PATCH 4/7] clean up --- go/cmd/paddlecloud/paddlecloud.go | 2 +- go/cmd/paddlectl/main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go/cmd/paddlecloud/paddlecloud.go b/go/cmd/paddlecloud/paddlecloud.go index c77d8bac..5128c6e9 100644 --- a/go/cmd/paddlecloud/paddlecloud.go +++ b/go/cmd/paddlecloud/paddlecloud.go @@ -13,7 +13,7 @@ import ( func main() { pfsmod.Config = config.ParseDefaultConfig() - paddlecloud.Config = config.ParseDefaultConfig() + paddlecloud.Config = pfsmod.Config subcommands.Register(subcommands.HelpCommand(), "") subcommands.Register(subcommands.CommandsCommand(), "") diff --git a/go/cmd/paddlectl/main.go b/go/cmd/paddlectl/main.go index 90844e08..ebc9a7d3 100644 --- a/go/cmd/paddlectl/main.go +++ b/go/cmd/paddlectl/main.go @@ -13,7 +13,7 @@ import ( func main() { pfsmod.Config = config.ParseDefaultConfig() - paddlecloud.Config = config.ParseDefaultConfig() + paddlecloud.Config = pfsmod.Config subcommands.Register(subcommands.HelpCommand(), "") subcommands.Register(subcommands.CommandsCommand(), "") From 66714a66957162d1519dcb24fc21e9334ee271a8 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Wed, 7 Feb 2018 02:02:52 +0000 Subject: [PATCH 5/7] cleanup --- go/cmd/paddlectl/main.go | 1 + go/paddlectl/submit.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/go/cmd/paddlectl/main.go b/go/cmd/paddlectl/main.go index ebc9a7d3..b2f6ab5e 100644 --- a/go/cmd/paddlectl/main.go +++ b/go/cmd/paddlectl/main.go @@ -5,6 +5,7 @@ import ( "flag" "os" + pfsmod "github.com/PaddlePaddle/cloud/go/filemanager/pfsmodules" "github.com/PaddlePaddle/cloud/go/paddlecloud" "github.com/PaddlePaddle/cloud/go/paddlectl" "github.com/PaddlePaddle/cloud/go/utils/config" diff --git a/go/paddlectl/submit.go b/go/paddlectl/submit.go index c7041239..99183077 100644 --- a/go/paddlectl/submit.go +++ b/go/paddlectl/submit.go @@ -22,7 +22,7 @@ const ( ) // Config is global config object for paddlectl commandline -var Config *config.SumitConfig = nil +var Config *config.SubmitConfig = nil // SubmitCmd define the subcommand of submitting paddle training jobs. type SubmitCmd struct { From 0e08a2f01452952f3eb179b979ee361ab766d979 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Wed, 7 Feb 2018 02:09:14 +0000 Subject: [PATCH 6/7] fix by golint --- go/filemanager/pfsmodules/upload.go | 2 +- go/paddlecloud/submit.go | 2 +- go/paddlectl/submit.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/filemanager/pfsmodules/upload.go b/go/filemanager/pfsmodules/upload.go index b58ebe27..9ceacab1 100644 --- a/go/filemanager/pfsmodules/upload.go +++ b/go/filemanager/pfsmodules/upload.go @@ -15,7 +15,7 @@ import ( ) // Config is global config object for pfs commandline -var Config *config.SubmitConfig = nil +var Config *config.SubmitConfig type uploadChunkResponse struct { Err string `json:"err"` diff --git a/go/paddlecloud/submit.go b/go/paddlecloud/submit.go index 6f6bdc25..36f19c57 100644 --- a/go/paddlecloud/submit.go +++ b/go/paddlecloud/submit.go @@ -21,7 +21,7 @@ const ( ) // Config is global config object for paddlecloud commandline -var Config *config.SubmitConfig = nil +var Config *config.SubmitConfig // SubmitCmd define the subcommand of submitting paddle training jobs. type SubmitCmd struct { diff --git a/go/paddlectl/submit.go b/go/paddlectl/submit.go index 99183077..9cda26f2 100644 --- a/go/paddlectl/submit.go +++ b/go/paddlectl/submit.go @@ -22,7 +22,7 @@ const ( ) // Config is global config object for paddlectl commandline -var Config *config.SubmitConfig = nil +var Config *config.SubmitConfig // SubmitCmd define the subcommand of submitting paddle training jobs. type SubmitCmd struct { From 36a6447d2b6ade82bb352c12a4b2371952945ce9 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Thu, 8 Feb 2018 01:47:13 +0000 Subject: [PATCH 7/7] fix test bug --- go/utils/restclient/token_test.go | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/go/utils/restclient/token_test.go b/go/utils/restclient/token_test.go index 46d2d0d0..a1221810 100644 --- a/go/utils/restclient/token_test.go +++ b/go/utils/restclient/token_test.go @@ -1,6 +1,7 @@ package restclient import ( + "errors" "fmt" "net" "net/http" @@ -38,12 +39,33 @@ func fakeServer() (*http.Server, int) { return srv, listener.Addr().(*net.TCPAddr).Port } +func mkdir_p(path string) error { + fi, err := os.Stat(path) + + if os.IsExist(err) { + if !fi.IsDir() { + return errors.New("exist a same name file") + } + + return nil + } + + if err := os.MkdirAll(path, 0700); err != nil { + return err + } + + return nil +} + func TestTokenParse(t *testing.T) { srv, port := fakeServer() defer srv.Shutdown(nil) // test token fetching - os.Remove(filepath.Join(pathutil.UserHomeDir(), ".paddle", "token_cache")) + path := filepath.Join(pathutil.UserHomeDir(), ".paddle") + require.Nil(t, mkdir_p(path), "mkdir ", path) + + os.Remove(filepath.Join(path, "token_cache")) tmpconf := &config.SubmitConfig{ActiveConfig: &config.SubmitConfigDataCenter{ Name: "test", Username: "testuser",