From 92047a102a0ea620176d12cda29c3f4a804f991d Mon Sep 17 00:00:00 2001 From: erwinvaneyk Date: Sat, 24 Mar 2018 13:59:14 +0100 Subject: [PATCH 1/4] bump fission 0.6.0 -> 0.6.1 Speed up builds in travis --- .Dockerignore | 1 + .travis.yml | 11 +++++++-- build/build-linux.sh | 1 + .../templates/deployment.yaml | 19 ++++++++-------- hack/deploy.sh | 12 +++++----- test/e2e/buildtest.sh | 1 - test/e2e/cleanup.sh | 21 +++++++++--------- test/e2e/install-clients.sh | 10 ++++----- .../__pycache__/testutils.cpython-36.pyc | Bin 680 -> 0 bytes test/e2e/travis-setup.sh | 17 ++++---------- test/e2e/utils.sh | 9 +++++++- 11 files changed, 54 insertions(+), 48 deletions(-) delete mode 100644 test/e2e/tests/__pycache__/testutils.cpython-36.pyc diff --git a/.Dockerignore b/.Dockerignore index 66a486b1..091fe3fd 100644 --- a/.Dockerignore +++ b/.Dockerignore @@ -2,3 +2,4 @@ vendor local Docs data +api \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 496fe78c..6fbeedb0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,12 +11,19 @@ cache: apt: true directories: - ${HOME}/.helm/ + - ${HOME}/.glide/ - /tmp/fission-workflow-ci/bin - ${GOPATH}/bin/ - ${GOPATH}/pkg/ env: - - KUBECONFIG=${HOME}/.kube/config PATH=/tmp/fission-workflow-ci/bin:${PATH} BIN_DIR=/tmp/fission-workflow-ci/bin + global: + - KUBECONFIG=${HOME}/.kube/config + - PATH=/tmp/fission-workflow-ci/bin:${PATH} + - BIN_DIR=/tmp/fission-workflow-ci/bin + - FISSION_VERSION=0.6.1 + - HELM_VERSION=2.8.2 + - KUBECTL_VERSION=1.9.6 services: - docker @@ -48,7 +55,7 @@ script: # Unit and Integration tests - test/runtests.sh # End-to-end tests -- test/e2e/buildtest.sh +- NOBUILD=y test/e2e/buildtest.sh after_script: - test/e2e/cleanup.sh diff --git a/build/build-linux.sh b/build/build-linux.sh index 28c86132..603ec3b4 100755 --- a/build/build-linux.sh +++ b/build/build-linux.sh @@ -1,4 +1,5 @@ #!/bin/sh +rm -f ./fission-workflows-bundle ./wfcli GOOS=linux GOARCH=386 go build github.com/fission/fission-workflows/cmd/fission-workflows-bundle/ GOOS=linux GOARCH=386 go build github.com/fission/fission-workflows/cmd/wfcli/ \ No newline at end of file diff --git a/charts/fission-workflows/templates/deployment.yaml b/charts/fission-workflows/templates/deployment.yaml index 724d8cd3..52c7a672 100644 --- a/charts/fission-workflows/templates/deployment.yaml +++ b/charts/fission-workflows/templates/deployment.yaml @@ -77,15 +77,16 @@ spec: version: 2 runtime: image: "{{ .Values.envImage }}:{{.Values.tag}}" - env: - - name: ES_NATS_URL - value: "nats://{{ .Values.nats.authToken }}@{{ .Values.nats.location }}.{{ .Values.fnenv.fission.ns }}:{{ .Values.nats.port }}" - - name: ES_NATS_CLUSTER - value: "{{ .Values.nats.cluster }}" - - name: FNENV_FISSION_CONTROLLER - value: "{{ .Values.fnenv.fission.controller }}.{{ .Values.fnenv.fission.ns }}" - - name: FNENV_FISSION_EXECUTOR - value: "{{ .Values.fnenv.fission.executor }}.{{ .Values.fnenv.fission.ns }}" + container: + env: + - name: ES_NATS_URL + value: "nats://{{ .Values.nats.authToken }}@{{ .Values.nats.location }}.{{ .Values.fnenv.fission.ns }}:{{ .Values.nats.port }}" + - name: ES_NATS_CLUSTER + value: "{{ .Values.nats.cluster }}" + - name: FNENV_FISSION_CONTROLLER + value: "{{ .Values.fnenv.fission.controller }}.{{ .Values.fnenv.fission.ns }}" + - name: FNENV_FISSION_EXECUTOR + value: "{{ .Values.fnenv.fission.executor }}.{{ .Values.fnenv.fission.ns }}" builder: image: "{{ .Values.buildEnvImage }}:{{.Values.tag}}" command: "defaultBuild" diff --git a/hack/deploy.sh b/hack/deploy.sh index 4da8b6ba..69a81247 100755 --- a/hack/deploy.sh +++ b/hack/deploy.sh @@ -5,8 +5,8 @@ # # Configs -FISSION_VERSION=0.6.0 -FISSION_WORKFLOWS_VERSION=0.2.0 +FISSION_VERSION=${FISSION_VERSION:-0.6.1} +WORKFLOWS_VERSION=${WORKFLOWS_VERSION:-0.2.0} DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" echo "Fission Workflows Deploy Script v1.4" @@ -26,9 +26,9 @@ sleep 10 # Install Fission Workflows if ! fission env get --name workflow >/dev/null 2>&1 ; then - echo "Installing Fission Workflows ${FISSION_WORKFLOWS_VERSION}..." - if [[ -z "${FISSION_WORKFLOWS_VERSION// }" ]] ; then - helm install --namespace fission -n fission-workflows fission-charts/fission-workflows --version ${FISSION_WORKFLOWS_VERSION} --wait + echo "Installing Fission Workflows ${WORKFLOWS_VERSION}..." + if [[ -z "${WORKFLOWS_VERSION// }" ]] ; then + helm install --namespace fission -n fission-workflows fission-charts/fission-workflows --version ${WORKFLOWS_VERSION} --wait else helm install --namespace fission -n fission-workflows fission-charts/fission-workflows --wait fi @@ -46,5 +46,5 @@ helm version --short -c printf "Helm " helm version --short -s echo "Fission: ${FISSION_VERSION}" -echo "Fission Workflows: ${FISSION_WORKFLOWS_VERSION}" +echo "Fission Workflows: ${WORKFLOWS_VERSION}" echo "---------------------------" diff --git a/test/e2e/buildtest.sh b/test/e2e/buildtest.sh index 6ca3c80b..67032e8c 100755 --- a/test/e2e/buildtest.sh +++ b/test/e2e/buildtest.sh @@ -19,7 +19,6 @@ NS_FUNCTION=fission-function NS_BUILDER=fission-builder fissionHelmId=fission fissionWorkflowsHelmId=fission-workflows -FISSION_VERSION=0.6.0 TEST_STATUS=0 TEST_LOGFILE_PATH=tests.log BIN_DIR="${BIN_DIR:-$HOME/testbin}" diff --git a/test/e2e/cleanup.sh b/test/e2e/cleanup.sh index 2044406b..f5d0567d 100755 --- a/test/e2e/cleanup.sh +++ b/test/e2e/cleanup.sh @@ -22,32 +22,33 @@ cleanup_fission_workflows() { } cleanup_fission() { - # Trigger deletion of all namespaces before waiting - for concurrency of deletion - emph "Forcing deletion of namespaces..." - kubectl delete ns/${NS} --now > /dev/null 2>&1 # Sometimes it is not deleted by helm delete - kubectl delete ns/${NS_BUILDER} --now > /dev/null 2>&1 # Sometimes it is not deleted by helm delete - kubectl delete ns/${NS_FUNCTION} --now > /dev/null 2>&1 # Sometimes it is not deleted by helm delete - cleanup_fission_workflows emph "Removing Fission deployment..." helm_uninstall_release ${fissionHelmId} emph "Removing custom resources..." clean_tpr_crd_resources || true + kubectl delete all --all -n ${NS} + kubectl delete all --all -n ${NS_FUNCTION} + kubectl delete all --all -n ${NS_BUILDER} + + + # Trigger deletion of all namespaces before waiting - for concurrency of deletion + emph "Forcing deletion of namespaces..." + kubectl delete ns/${NS} --now > /dev/null 2>&1 # Sometimes it is not deleted by helm delete + kubectl delete ns/${NS_BUILDER} --now > /dev/null 2>&1 # Sometimes it is not deleted by helm delete + kubectl delete ns/${NS_FUNCTION} --now > /dev/null 2>&1 # Sometimes it is not deleted by helm delete # Wait until all namespaces are actually deleted! - sleep 10 emph "Awaiting deletion of namespaces..." verify_ns_deleted() { kubectl delete ns/${1} --now 2>&1 | grep -qv "Error from server (Conflict):" } # Namespaces sometimes take a long time to delete for some reason + sleep 10 RETRY_LIMIT=10 RETRY_DELAY=10 retry verify_ns_deleted ${NS_BUILDER} RETRY_LIMIT=10 RETRY_DELAY=10 retry verify_ns_deleted ${NS} RETRY_LIMIT=10 RETRY_DELAY=10 retry verify_ns_deleted ${NS_FUNCTION} - - emph "Cleaning up local filesystem..." - rm -f ./fission-workflows-bundle ./wfcli } reset_fission_crd_resources() { diff --git a/test/e2e/install-clients.sh b/test/e2e/install-clients.sh index ff42fe2b..af18556b 100755 --- a/test/e2e/install-clients.sh +++ b/test/e2e/install-clients.sh @@ -5,9 +5,9 @@ set -euo pipefail . $(dirname $0)/utils.sh BIN_DIR=${BIN_DIR:-/tmp/fission-workflow-ci/bin} -HELM_VERSION=2.8.2 -KUBECTL_VERSION=1.9.6 -FISSION_VERSION=0.6.0 +HELM_VERSION=${HELM_VERSION:-2.8.2} +KUBECTL_VERSION=${KUBECTL_VERSION:-1.9.6} +FISSION_VERSION=${FISSION_VERSION:-0.6.1} # Install kubectl if ! kubectl version -c 2>/dev/null | grep ${KUBECTL_VERSION} >/dev/null; then @@ -25,7 +25,7 @@ which kubectl if ! helm version -c 2>/dev/null | grep ${HELM_VERSION} >/dev/null; then emph "Installing Helm ${HELM_VERSION} to ${BIN_DIR}/helm..." curl -sLO https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VERSION}-linux-amd64.tar.gz - tar xzvf helm-*.tar.gz + tar xzvf helm-*.tar.gz >/dev/null chmod +x linux-amd64/helm mv -f linux-amd64/helm ${BIN_DIR}/helm else @@ -44,6 +44,4 @@ else fi which fission -# TODO install gcloud - emph "Clients installed in ${BIN_DIR}" \ No newline at end of file diff --git a/test/e2e/tests/__pycache__/testutils.cpython-36.pyc b/test/e2e/tests/__pycache__/testutils.cpython-36.pyc deleted file mode 100644 index e6ab081c9527e0c4e811589cd78163501934f2d0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 680 zcmYjP%W4}j6qV-XiNS4JXcncE*_d<|&OqB$2%*7_q0qVu+nXt+BqQ0DA2a4&4aQ-- zO1Axw{!70=+ph8n&8k;+LoGPx>RjD>^g!<#jryNo>-4?L*l)HsNV5+Fd`%>n;B%Jl zB2T#pgeUwvo;rf}SzNoP8rHi%BkpFN6*K}uNN5vqLZrz>I8$D+TYf~~OgmGzVinik z)Ten+Il>j*k6=UY8^7|VH4)4lI2W~zPu*eVKla;F0mKFjFe_jj!@$sZD@d6#t?GFg2|f(k4QgXN6GtI1RcNA3t4o zfYSv`tv_tHFZ=zQxNiNwHu2luCT~FM67q;TRBq~6UN51@ zB%*Pl%#Dj{1p8KL&>wWZ3/dev/null 2>&1 | grep "current-context: gke_fission-ci_us-central1-a_fission-workflows-ci-1" ; +then emph "Connecting to gcloud cluster..." gcloud container clusters get-credentials fission-workflows-ci-1 --zone us-central1-a --project fission-ci fi @@ -91,11 +89,4 @@ sleep 5 retry fission fn list echo fission --version -emph "Fission deployed!" - -# Verify clients -ls -l ${BIN_DIR} -kubectl version -helm version -fission --version -emph "All clients available!" \ No newline at end of file +emph "Fission deployed!" \ No newline at end of file diff --git a/test/e2e/utils.sh b/test/e2e/utils.sh index 70fd35fd..16d40e0f 100644 --- a/test/e2e/utils.sh +++ b/test/e2e/utils.sh @@ -132,7 +132,7 @@ function retry { sleep ${RETRY_DELAY}; else >&2 echo "The command has failed after $n attempts." - exit 1; + return 1; fi } done @@ -205,19 +205,26 @@ dump_system_info() { echo "--- System Info ---" echo "--- go ---" go version + echo echo "--- python ---" python --version python3 --version + echo echo "--- docker ---" docker version + echo echo "--- kubectl ---" kubectl version + echo echo "--- Helm ---" helm version + echo echo "--- fission ---" fission -v + echo echo "--- wfcli ---" wfcli version + echo echo "--- End System Info ---" } From 27294d1f35644117253d87925ec435d9353f0396 Mon Sep 17 00:00:00 2001 From: erwinvaneyk Date: Mon, 26 Mar 2018 18:35:05 +0200 Subject: [PATCH 2/4] Changed wfcli to use consistent client connections with Fission only when needed --- cmd/wfcli/admin.go | 30 +++++------- cmd/wfcli/config.go | 12 ++--- cmd/wfcli/invocation.go | 103 +++++++++++++++------------------------ cmd/wfcli/main.go | 50 +++++++++++++------ cmd/wfcli/parse.go | 12 ++--- cmd/wfcli/portforward.go | 35 +++++++++++++ cmd/wfcli/status.go | 14 ++---- cmd/wfcli/validate.go | 10 ++-- cmd/wfcli/version.go | 13 ++--- cmd/wfcli/workflow.go | 35 +++++-------- 10 files changed, 160 insertions(+), 154 deletions(-) diff --git a/cmd/wfcli/admin.go b/cmd/wfcli/admin.go index 8113154a..02c21c97 100644 --- a/cmd/wfcli/admin.go +++ b/cmd/wfcli/admin.go @@ -1,46 +1,38 @@ package main import ( - "context" - "net/http" - - "github.com/fission/fission-workflows/pkg/apiserver/httpclient" "github.com/urfave/cli" ) var cmdAdmin = cli.Command{ - Name: "admin", + Name: "Admin", Usage: "Administrative commands", Subcommands: []cli.Command{ cmdStatus, cmdVersion, { Name: "halt", - Usage: "Stop the workflow engine from evaluating anything", - Action: func(c *cli.Context) error { - ctx := context.TODO() - url := parseUrl(c.GlobalString("url")) - admin := httpclient.NewAdminApi(url.String(), http.Client{}) - err := admin.Halt(ctx) + Usage: "Stop the Workflow engine from evaluating anything", + Action: commandContext(func(ctx Context) error { + client := getClient(ctx) + err := client.Admin.Halt(ctx) if err != nil { panic(err) } return nil - }, + }), }, { Name: "resume", - Usage: "Resume the workflow engine evaluations", - Action: func(c *cli.Context) error { - ctx := context.TODO() - url := parseUrl(c.GlobalString("url")) - admin := httpclient.NewAdminApi(url.String(), http.Client{}) - err := admin.Resume(ctx) + Usage: "Resume the Workflow engine evaluations", + Action: commandContext(func(ctx Context) error { + client := getClient(ctx) + err := client.Admin.Resume(ctx) if err != nil { panic(err) } return nil - }, + }), }, }, } diff --git a/cmd/wfcli/config.go b/cmd/wfcli/config.go index 73891fb8..6d3616fe 100644 --- a/cmd/wfcli/config.go +++ b/cmd/wfcli/config.go @@ -9,14 +9,14 @@ import ( var cmdConfig = cli.Command{ Name: "config", Usage: "Print wfcli config", - Action: func(c *cli.Context) error { + Action: commandContext(func(ctx Context) error { fmt.Println("cli:") - for _, flag := range c.GlobalFlagNames() { - fmt.Printf(" %s: %v\n", flag, c.GlobalGeneric(flag)) + for _, flag := range ctx.GlobalFlagNames() { + fmt.Printf(" %s: %v\n", flag, ctx.GlobalGeneric(flag)) } - for _, flag := range c.FlagNames() { - fmt.Printf(" %s: %v\n", flag, c.Generic(flag)) + for _, flag := range ctx.FlagNames() { + fmt.Printf(" %s: %v\n", flag, ctx.Generic(flag)) } return nil - }, + }), } diff --git a/cmd/wfcli/invocation.go b/cmd/wfcli/invocation.go index 74eca6ce..3ef7e60d 100644 --- a/cmd/wfcli/invocation.go +++ b/cmd/wfcli/invocation.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - "net/http" "os" "sort" "time" @@ -16,13 +15,13 @@ import ( ) var cmdInvocation = cli.Command{ - Name: "invocation", - Aliases: []string{"wi", "invocations", "workflow-invocation", "wfi"}, + Name: "Invocation", + Aliases: []string{"wi", "invocations", "Workflow-Invocation", "wfi"}, Usage: "Workflow Invocation-related commands", Subcommands: []cli.Command{ { Name: "get", - Usage: "get ", + Usage: "get ", Flags: []cli.Flag{ cli.DurationFlag{ Name: "history", @@ -30,21 +29,16 @@ var cmdInvocation = cli.Command{ Value: time.Duration(1) * time.Hour, }, }, - Action: func(c *cli.Context) error { - //u := parseUrl(c.GlobalString("url")) - //client := createTransportClient(u) - //wfiApi := workflow_invocation_api.New(client, strfmt.Default) - ctx := context.TODO() - url := parseUrl(c.GlobalString("url")) - wfiApi := httpclient.NewInvocationApi(url.String(), http.Client{}) - switch c.NArg() { + Action: commandContext(func(ctx Context) error { + client := getClient(ctx) + switch ctx.NArg() { case 0: - since := c.Duration("history") - invocationsList(os.Stdout, wfiApi, time.Now().Add(-since)) + since := ctx.Duration("history") + invocationsList(os.Stdout, client.Invocation, time.Now().Add(-since)) case 1: - // Get Workflow invocation - wfiId := c.Args().Get(0) - wfi, err := wfiApi.Get(ctx, wfiId) + // Get Workflow Invocation + wfiId := ctx.Args().Get(0) + wfi, err := client.Invocation.Get(ctx, wfiId) if err != nil { panic(err) } @@ -56,15 +50,15 @@ var cmdInvocation = cli.Command{ case 2: fallthrough default: - wfiId := c.Args().Get(0) - taskId := c.Args().Get(1) - wfi, err := wfiApi.Get(ctx, wfiId) + wfiId := ctx.Args().Get(0) + taskId := ctx.Args().Get(1) + wfi, err := client.Invocation.Get(ctx, wfiId) if err != nil { panic(err) } ti, ok := wfi.Status.Tasks[taskId] if !ok { - fmt.Println("Task invocation not found.") + fmt.Println("Task Invocation not found.") return nil } b, err := yaml.Marshal(ti) @@ -75,30 +69,25 @@ var cmdInvocation = cli.Command{ } return nil - }, + }), }, { Name: "cancel", - Usage: "cancel ", - Action: func(c *cli.Context) error { - wfiId := c.Args().Get(0) - //u := parseUrl(c.GlobalString("url")) - //client := createTransportClient(u) - //wfiApi := workflow_invocation_api.New(client, strfmt.Default) - ctx := context.TODO() - url := parseUrl(c.GlobalString("url")) - wfiApi := httpclient.NewInvocationApi(url.String(), http.Client{}) - err := wfiApi.Cancel(ctx, wfiId) + Usage: "cancel ", + Action: commandContext(func(ctx Context) error { + client := getClient(ctx) + wfiId := ctx.Args().Get(0) + err := client.Invocation.Cancel(ctx, wfiId) if err != nil { panic(err) } return nil - }, + }), }, { // TODO support input Name: "invoke", - Usage: "invoke ", + Usage: "invoke ", Flags: []cli.Flag{ cli.StringSliceFlag{ Name: "input, i", @@ -109,20 +98,15 @@ var cmdInvocation = cli.Command{ Usage: "Invoke synchronously", }, }, - Action: func(c *cli.Context) error { - wfId := c.Args().Get(0) - //u := parseUrl(c.GlobalString("url")) - //client := createTransportClient(u) - //wfiApi := workflow_invocation_api.New(client, strfmt.Default) - ctx := context.TODO() - url := parseUrl(c.GlobalString("url")) - wfiApi := httpclient.NewInvocationApi(url.String(), http.Client{}) + Action: commandContext(func(ctx Context) error { + client := getClient(ctx) + wfId := ctx.Args().Get(0) spec := &types.WorkflowInvocationSpec{ WorkflowId: wfId, Inputs: map[string]*types.TypedValue{}, } - if c.Bool("sync") { - resp, err := wfiApi.InvokeSync(ctx, spec) + if ctx.Bool("sync") { + resp, err := client.Invocation.InvokeSync(ctx, spec) if err != nil { panic(err) } @@ -132,39 +116,32 @@ var cmdInvocation = cli.Command{ } fmt.Println(string(bs)) } else { - resp, err := wfiApi.Invoke(ctx, spec) + resp, err := client.Invocation.Invoke(ctx, spec) if err != nil { panic(err) } fmt.Println(resp.Id) } return nil - }, + }), }, { Name: "status", - Usage: "status ", - Action: func(c *cli.Context) error { - if c.NArg() < 1 { - fmt.Println("Need workflow invocation id") + Usage: "status ", + Action: commandContext(func(ctx Context) error { + if ctx.NArg() < 1 { + fmt.Println("Need Workflow Invocation id") return nil } - wfiId := c.Args().Get(0) - //u := parseUrl(c.GlobalString("url")) - //client := createTransportClient(u) - //wfApi := workflow_api.New(client, strfmt.Default) - //wfiApi := workflow_invocation_api.New(client, strfmt.Default) - ctx := context.TODO() - url := parseUrl(c.GlobalString("url")) - wfiApi := httpclient.NewInvocationApi(url.String(), http.Client{}) - wfApi := httpclient.NewWorkflowApi(url.String(), http.Client{}) - - wfi, err := wfiApi.Get(ctx, wfiId) + client := getClient(ctx) + wfiId := ctx.Args().Get(0) + + wfi, err := client.Invocation.Get(ctx, wfiId) if err != nil { panic(err) } - wf, err := wfApi.Get(ctx, wfi.Spec.WorkflowId) + wf, err := client.Workflow.Get(ctx, wfi.Spec.WorkflowId) if err != nil { panic(err) } @@ -190,7 +167,7 @@ var cmdInvocation = cli.Command{ table(os.Stdout, []string{"TASK", "STATUS", "STARTED", "UPDATED"}, rows) return nil - }, + }), }, }, } diff --git a/cmd/wfcli/main.go b/cmd/wfcli/main.go index 4cebeb25..02029689 100644 --- a/cmd/wfcli/main.go +++ b/cmd/wfcli/main.go @@ -7,25 +7,15 @@ import ( "os" "strings" "text/tabwriter" + "time" "github.com/fission/fission-workflows/pkg/version" + "github.com/sirupsen/logrus" "github.com/urfave/cli" ) // This is a prototype of the CLI (and will be integrated into the Fission CLI eventually). func main() { - // fetch the FISSION_URL env variable. If not set, port-forward to controller. - var value string - fissionUrl := os.Getenv("FISSION_URL") - if len(fissionUrl) == 0 { - fissionNamespace := getFissionNamespace() - kubeConfig := getKubeConfigPath() - localPort := setupPortForward(kubeConfig, fissionNamespace, "application=fission-api") - value = "http://127.0.0.1:" + localPort - fmt.Printf("Forwarded Fission API to %s.\n", value) - } else { - value = fissionUrl - } app := cli.NewApp() app.Author = "Erwin van Eyk" @@ -38,7 +28,6 @@ func main() { app.Flags = []cli.Flag{ cli.StringFlag{ Name: "url, u", - Value: value, EnvVar: "FISSION_URL", Usage: "Url to the Fission apiserver", }, @@ -57,7 +46,6 @@ func main() { cmdAdmin, cmdVersion, } - app.Run(os.Args) } @@ -89,3 +77,37 @@ func parseUrl(rawUrl string) *url.URL { } return u } + +type Context struct { + *cli.Context +} + +func (c Context) Deadline() (deadline time.Time, ok bool) { + return +} + +func (c Context) Done() <-chan struct{} { + return nil +} + +func (c Context) Err() error { + return nil +} + +func (c Context) Value(key interface{}) interface{} { + if s, ok := key.(string); ok { + return c.Generic(s) + } + return nil +} + +func commandContext(fn func(c Context) error) func(c *cli.Context) error { + return func(c *cli.Context) error { + if c.GlobalBool("debug") { + logrus.SetLevel(logrus.DebugLevel) + } else { + logrus.SetLevel(logrus.InfoLevel) + } + return fn(Context{c}) + } +} diff --git a/cmd/wfcli/parse.go b/cmd/wfcli/parse.go index 6a9a2c67..5768f4d7 100644 --- a/cmd/wfcli/parse.go +++ b/cmd/wfcli/parse.go @@ -23,18 +23,18 @@ var cmdParse = cli.Command{ }, }, Description: "Read YAML definitions to the executable JSON format (deprecated)", - Action: func(c *cli.Context) error { + Action: commandContext(func(ctx Context) error { - if c.NArg() == 0 { - panic("Need a path to a yaml workflow definition") + if ctx.NArg() == 0 { + panic("Need a path to a yaml Workflow definition") } - parserType := c.String("type") + parserType := ctx.String("type") if parserType != "" && !parse.Supports(parserType) { fmt.Printf("Unknown parser '%s'; will try all parsers.", parserType) } - for _, path := range c.Args() { + for _, path := range ctx.Args() { fnName := strings.TrimSpace(path) @@ -51,7 +51,7 @@ var cmdParse = cli.Command{ println(toFormattedJson(wfSpec)) } return nil - }, + }), } func toFormattedJson(spec *types.WorkflowSpec) string { diff --git a/cmd/wfcli/portforward.go b/cmd/wfcli/portforward.go index d556f3dc..82c75968 100644 --- a/cmd/wfcli/portforward.go +++ b/cmd/wfcli/portforward.go @@ -3,12 +3,15 @@ package main import ( "fmt" "net" + "net/http" "os" "path/filepath" "strconv" "strings" "time" + "github.com/fission/fission-workflows/pkg/apiserver/httpclient" + "github.com/sirupsen/logrus" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" _ "k8s.io/client-go/plugin/pkg/client/auth" @@ -18,6 +21,38 @@ import ( "k8s.io/client-go/tools/remotecommand" ) +type client struct { + Admin *httpclient.AdminApi + Workflow *httpclient.WorkflowApi + Invocation *httpclient.InvocationApi +} + +func getClient(ctx Context) client { + + url := ctx.GlobalString("url") + + // fetch the FISSION_URL env variable. If not set, port-forward to controller. + if len(url) == 0 { + fissionUrl := os.Getenv("FISSION_URL") + if len(fissionUrl) == 0 { + fissionNamespace := getFissionNamespace() + kubeConfig := getKubeConfigPath() + localPort := setupPortForward(kubeConfig, fissionNamespace, "application=fission-api") + url = "http://127.0.0.1:" + localPort + logrus.Debugf("Forwarded Fission API to %s.", url) + } else { + url = fissionUrl + } + } + + httpClient := http.Client{} + return client{ + Admin: httpclient.NewAdminApi(url, httpClient), + Workflow: httpclient.NewWorkflowApi(url, httpClient), + Invocation: httpclient.NewInvocationApi(url, httpClient), + } +} + func getFissionNamespace() string { fissionNamespace := os.Getenv("FISSION_NAMESPACE") return fissionNamespace diff --git a/cmd/wfcli/status.go b/cmd/wfcli/status.go index 866abd50..7c3b2d4a 100644 --- a/cmd/wfcli/status.go +++ b/cmd/wfcli/status.go @@ -1,11 +1,8 @@ package main import ( - "context" "fmt" - "net/http" - "github.com/fission/fission-workflows/pkg/apiserver/httpclient" "github.com/urfave/cli" ) @@ -13,17 +10,14 @@ var cmdStatus = cli.Command{ Name: "status", Aliases: []string{"s"}, Usage: "Check cluster status", - Action: func(c *cli.Context) error { - ctx := context.TODO() - url := parseUrl(c.GlobalString("url")) - admin := httpclient.NewAdminApi(url.String(), http.Client{}) - - resp, err := admin.Status(ctx) + Action: commandContext(func(ctx Context) error { + client := getClient(ctx) + resp, err := client.Admin.Status(ctx) if err != nil { panic(err) } fmt.Printf(resp.Status) return nil - }, + }), } diff --git a/cmd/wfcli/validate.go b/cmd/wfcli/validate.go index a7794fe7..2fbe0de0 100644 --- a/cmd/wfcli/validate.go +++ b/cmd/wfcli/validate.go @@ -14,15 +14,15 @@ import ( var cmdValidate = cli.Command{ Name: "validate", Usage: "Validate ", - Description: "Validate a workflow", - Action: func(c *cli.Context) error { + Description: "Validate a Workflow", + Action: commandContext(func(ctx Context) error { // Get path from args - if c.NArg() == 0 { + if ctx.NArg() == 0 { fail("No file provided.") } var failed bool - for _, path := range c.Args() { + for _, path := range ctx.Args() { printErr := func(msg string) { fmt.Fprintf(os.Stderr, "%s: %s\n", path, msg) @@ -61,5 +61,5 @@ var cmdValidate = cli.Command{ } return nil - }, + }), } diff --git a/cmd/wfcli/version.go b/cmd/wfcli/version.go index 2c8dc182..1047ba7d 100644 --- a/cmd/wfcli/version.go +++ b/cmd/wfcli/version.go @@ -1,31 +1,26 @@ package main import ( - "context" "fmt" - "net/http" - "github.com/fission/fission-workflows/pkg/apiserver/httpclient" "github.com/fission/fission-workflows/pkg/version" "github.com/urfave/cli" ) -var versionPrinter = func(c *cli.Context) error { +var versionPrinter = commandContext(func(ctx Context) error { // Print client version fmt.Printf("client: %s\n", version.VERSION) // Print server version - ctx := context.TODO() - url := parseUrl(c.GlobalString("url")) - admin := httpclient.NewAdminApi(url.String(), http.Client{}) - resp, err := admin.Version(ctx) + client := getClient(ctx) + resp, err := client.Admin.Version(ctx) if err != nil { fmt.Printf("server: failed to get version (%v)\n", err) } else { fmt.Printf("server: %s\n", resp.Version) } return nil -} +}) var cmdVersion = cli.Command{ Name: "version", diff --git a/cmd/wfcli/workflow.go b/cmd/wfcli/workflow.go index dd6b5eab..b9f7674d 100644 --- a/cmd/wfcli/workflow.go +++ b/cmd/wfcli/workflow.go @@ -1,38 +1,29 @@ package main import ( - "context" "fmt" - "net/http" "os" "sort" - "github.com/fission/fission-workflows/pkg/apiserver/httpclient" "github.com/fission/fission-workflows/pkg/parse/yaml" "github.com/urfave/cli" ) var cmdWorkflow = cli.Command{ - Name: "workflow", + Name: "Workflow", Aliases: []string{"wf", "workflows"}, Usage: "Workflow-related commands", Subcommands: []cli.Command{ { Name: "get", - Usage: "get ", - Action: func(c *cli.Context) error { - //u := parseUrl(c.GlobalString("url")) - //client := createTransportClient(u) - //wfApi := workflow_api.New(client, strfmt.Default) - ctx := context.TODO() - url := parseUrl(c.GlobalString("url")) - wfApi := httpclient.NewWorkflowApi(url.String(), http.Client{}) + Usage: "get ", + Action: commandContext(func(ctx Context) error { + client := getClient(ctx) - switch c.NArg() { + switch ctx.NArg() { case 0: // List workflows - resp, err := wfApi.List(ctx) - //resp, err := wf.(workflow_api.NewWfListParams()) + resp, err := client.Workflow.List(ctx) if err != nil { panic(err) } @@ -40,7 +31,7 @@ var cmdWorkflow = cli.Command{ sort.Strings(wfs) var rows [][]string for _, wfId := range wfs { - wf, err := wfApi.Get(ctx, wfId) + wf, err := client.Workflow.Get(ctx, wfId) if err != nil { panic(err) } @@ -53,9 +44,9 @@ var cmdWorkflow = cli.Command{ table(os.Stdout, []string{"ID", "NAME", "STATUS", "CREATED", "UPDATED"}, rows) case 1: // Get Workflow - wfId := c.Args().Get(0) + wfId := ctx.Args().Get(0) println(wfId) - wf, err := wfApi.Get(ctx, wfId) + wf, err := client.Workflow.Get(ctx, wfId) if err != nil { panic(err) } @@ -68,9 +59,9 @@ var cmdWorkflow = cli.Command{ // Get Workflow task fallthrough default: - wfId := c.Args().Get(0) - taskId := c.Args().Get(1) - wf, err := wfApi.Get(ctx, wfId) + wfId := ctx.Args().Get(0) + taskId := ctx.Args().Get(1) + wf, err := client.Workflow.Get(ctx, wfId) if err != nil { panic(err) } @@ -87,7 +78,7 @@ var cmdWorkflow = cli.Command{ } return nil - }, + }), }, }, } From c4e857e1d499e0f07c5fffce07a252e88af1116b Mon Sep 17 00:00:00 2001 From: erwinvaneyk Date: Mon, 26 Mar 2018 18:35:33 +0200 Subject: [PATCH 3/4] Fixed bug in httpclient: crashing on empty response body --- pkg/apiserver/httpclient/httpclient.go | 52 ++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/pkg/apiserver/httpclient/httpclient.go b/pkg/apiserver/httpclient/httpclient.go index b2e7d712..79c870f1 100644 --- a/pkg/apiserver/httpclient/httpclient.go +++ b/pkg/apiserver/httpclient/httpclient.go @@ -3,6 +3,7 @@ package httpclient import ( "bytes" + "errors" "fmt" "io" "io/ioutil" @@ -11,6 +12,15 @@ import ( "github.com/golang/protobuf/jsonpb" "github.com/golang/protobuf/proto" + "github.com/sirupsen/logrus" +) + +var ( + ErrResponseError = errors.New("response error") + ErrRequestCreate = errors.New("failed to create request") + ErrRequestSend = errors.New("failed to send request") + ErrSerialize = errors.New("failed to serialize input") + ErrDeserialize = errors.New("failed to deserialize input") ) var defaultHttpClient = http.Client{} @@ -29,32 +39,58 @@ func call(method string, url string, in proto.Message, out proto.Message) error if in != nil { err := toJson(buf, in) if err != nil { - return err + return fmt.Errorf("%v: %v", ErrSerialize, err) + } + } + if logrus.GetLevel() == logrus.DebugLevel { + logrus.Debugf("--> %s - %s", method, url) + if in != nil { + buf := bytes.NewBuffer(nil) + err := toJson(buf, in) + if err != nil { + logrus.Errorf("Failed to jsonify debug data: %v", err) + } + data, err := ioutil.ReadAll(buf) + if err != nil { + logrus.Errorf("Failed to read debug data: %v", err) + } + logrus.Debug("body: '%v'", data) } } req, err := http.NewRequest(method, url, buf) if err != nil { - return err + return fmt.Errorf("%v: %v", ErrRequestCreate, err) } req.Header.Set("Content-Type", "application/json") resp, err := defaultHttpClient.Do(req) if err != nil { - return err + return fmt.Errorf("%v: %v", ErrRequestSend, err) + } + if logrus.GetLevel() == logrus.DebugLevel { + logrus.Debugf("<-- %s - %s", resp.Status, url) + if resp.Body != nil { + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + logrus.Errorf("Failed to read debug body", err) + } + logrus.Debugf("Body: '%v'", string(data)) + } } if resp.StatusCode >= 400 { data, _ := ioutil.ReadAll(resp.Body) resp.Body.Close() - return fmt.Errorf("request failed - %s: %s", resp.Status, strings.TrimSpace(string(data))) + return fmt.Errorf("%v (%s): %s", ErrResponseError, resp.Status, strings.TrimSpace(string(data))) } - if out != nil { + if out != nil && resp.ContentLength != 0 { + err = fromJson(resp.Body, out) if err != nil { - return err + return fmt.Errorf("%v: %v", ErrDeserialize, err) } - err = resp.Body.Close() + resp.Body.Close() if err != nil { - return err + logrus.Errorf("Failed to close response body: %v", err) } } return nil From 46192e801f22d17f4256c2b846f5993a3b0b288b Mon Sep 17 00:00:00 2001 From: erwinvaneyk Date: Mon, 26 Mar 2018 18:43:08 +0200 Subject: [PATCH 4/4] Moved Fission proxy path prefix to wfcli client --- cmd/wfcli/admin.go | 2 +- cmd/wfcli/invocation.go | 4 ++-- cmd/wfcli/main.go | 6 ++++++ cmd/wfcli/portforward.go | 6 +++++- cmd/wfcli/workflow.go | 2 +- pkg/apiserver/httpclient/httpclient.go | 7 +++---- 6 files changed, 18 insertions(+), 9 deletions(-) diff --git a/cmd/wfcli/admin.go b/cmd/wfcli/admin.go index 02c21c97..7693d8b7 100644 --- a/cmd/wfcli/admin.go +++ b/cmd/wfcli/admin.go @@ -5,7 +5,7 @@ import ( ) var cmdAdmin = cli.Command{ - Name: "Admin", + Name: "admin", Usage: "Administrative commands", Subcommands: []cli.Command{ cmdStatus, diff --git a/cmd/wfcli/invocation.go b/cmd/wfcli/invocation.go index 3ef7e60d..be05f16a 100644 --- a/cmd/wfcli/invocation.go +++ b/cmd/wfcli/invocation.go @@ -15,9 +15,9 @@ import ( ) var cmdInvocation = cli.Command{ - Name: "Invocation", + Name: "invocation", Aliases: []string{"wi", "invocations", "Workflow-Invocation", "wfi"}, - Usage: "Workflow Invocation-related commands", + Usage: "Workflow invocation-related commands", Subcommands: []cli.Command{ { Name: "get", diff --git a/cmd/wfcli/main.go b/cmd/wfcli/main.go index 02029689..b24a29df 100644 --- a/cmd/wfcli/main.go +++ b/cmd/wfcli/main.go @@ -31,6 +31,12 @@ func main() { EnvVar: "FISSION_URL", Usage: "Url to the Fission apiserver", }, + cli.StringFlag{ + Name: "path-prefix", + EnvVar: "FISSION_PATH_PREFIX", + Value: "/proxy/workflows-apiserver", + Usage: "The path to prepend each of the commands", + }, cli.BoolFlag{ Name: "debug, d", EnvVar: "WFCLI_DEBUG", diff --git a/cmd/wfcli/portforward.go b/cmd/wfcli/portforward.go index 82c75968..5d82fb84 100644 --- a/cmd/wfcli/portforward.go +++ b/cmd/wfcli/portforward.go @@ -44,7 +44,11 @@ func getClient(ctx Context) client { url = fissionUrl } } - + path := ctx.GlobalString("path-prefix") + if path[0] != '/' { + path = "/" + path + } + url = url + strings.TrimSuffix(path, "/") httpClient := http.Client{} return client{ Admin: httpclient.NewAdminApi(url, httpClient), diff --git a/cmd/wfcli/workflow.go b/cmd/wfcli/workflow.go index b9f7674d..e88589f8 100644 --- a/cmd/wfcli/workflow.go +++ b/cmd/wfcli/workflow.go @@ -10,7 +10,7 @@ import ( ) var cmdWorkflow = cli.Command{ - Name: "Workflow", + Name: "workflow", Aliases: []string{"wf", "workflows"}, Usage: "Workflow-related commands", Subcommands: []cli.Command{ diff --git a/pkg/apiserver/httpclient/httpclient.go b/pkg/apiserver/httpclient/httpclient.go index 79c870f1..891e7230 100644 --- a/pkg/apiserver/httpclient/httpclient.go +++ b/pkg/apiserver/httpclient/httpclient.go @@ -1,4 +1,4 @@ -// Package httpclient is a lightweight implementation of the HTTP gateway. +// Package httpclient is a lightweight implementation of a client for the HTTP gateway. package httpclient import ( @@ -71,7 +71,7 @@ func call(method string, url string, in proto.Message, out proto.Message) error if resp.Body != nil { data, err := ioutil.ReadAll(resp.Body) if err != nil { - logrus.Errorf("Failed to read debug body", err) + logrus.Errorf("Failed to read debug body: %v", err) } logrus.Debugf("Body: '%v'", string(data)) } @@ -101,7 +101,6 @@ type BaseApi struct { client http.Client } -// TODO remove hard-coded proxy func (api *BaseApi) formatUrl(path string) string { - return api.endpoint + "/proxy/workflows-apiserver" + path + return api.endpoint + path }