Merge pull request #221 from fission/standalone-proxy

Extract environment proxy

.gitignore

@@ -8,11 +8,14 @@ __pycache__/
 
 revive.toml
 
-fission-workflows-bundle
-fission-workflows-bundle-linux
-fission-workflows-bundle-osx
-fission-workflows-bundle-windows
-fission-workflows
-fission-workflows-osx
-fission-workflows-linux
-fission-workflows-windows
+/fission-workflows-bundle
+/fission-workflows-bundle-linux
+/fission-workflows-bundle-osx
+/fission-workflows-bundle-windows
+/fission-workflows
+/fission-workflows-osx
+/fission-workflows-linux
+/fission-workflows-windows
+/fission-workflows-proxy-windows
+/fission-workflows-proxy-osx
+/fission-workflows-proxy

Docs/development.md

@@ -0,0 +1,32 @@
+# Development and Testing of Fission Workflows 
+
+## Development
+
+To test with local builds of the workflow engine in the Kubernetes cluster in the current context. First [install a 
+workflow version to your cluster](../INSTALL.md). After that you can swap the deployed workflow engine with a local one 
+using [telepresence](https://telepresence.io):
+
+```bash
+telepresence --method=vpn-tcp --namespace fission --swap-deployment workflows:workflows --expose 5555 --expose 8080
+```
+
+## Testing
+
+To run local unit and integration tests:
+
+```bash
+bash tests/runtests.sh
+```
+
+If you have an instance of Workflows deployed in the Kubernetes cluster in the current context, you can also run 
+end-to-end tests (the same that are used by the CI):
+
+```bash
+bash tests/e2e/runtests.sh
+```
+
+Since each e2e test is standalone, you can also run a single e2e test:
+
+```bash
+bash tests/e2e/tests/test_inputs.sh
+```

Docs/instrumentation.md

@@ -37,20 +37,39 @@ annotations:
 (In the near future, this step will be done by Fission automatically.)
 
 Now, to access metrics in the Prometheus dashboard you just need to exposes the prometheus-server in the `monitoring` 
-namespace or access the cluserIP from within the cluster.  
+namespace or access the clusterIP from within the cluster (for example by using [telepresence](https://telepresence
+.io/))
 
 ### Prometheus NATS exporter
 Given that NATS streaming plays an important role in the workflow system, it is also useful to collect the metrics of 
 NATS into prometheus. Although not directly implemented in the NATS deployments, there is the 
 [Prometheus NATS exporter](https://github.com/nats-io/prometheus-nats-exporter) as a separate module to install.
 
-## Grafana
+### Grafana
 A common way to visualize the metrics collected with Prometheus is to use Grafana to create and share graphs and 
 other types of visualizations.
 
 ```bash
-helm install --namespace monitoring --name prometheus stable/grafana
+helm install --namespace monitoring --name grafana stable/grafana
 ```
 
+Follow the instructions provided by the notes of the grafana helm chart to get the login details and exposing the 
+Grafana dashboard locally.
+
 In the future, we will provide a pre-built Grafana dashboard with useful graphs to provide you insight into the 
-system, without needing to build dashboards yourself.
+system, without needing to build dashboards yourself.
+
+## OpenTracing / Jaeger
+
+Fission Workflows supports distributed tracing using the [OpenTracing](http://opentracing.io/) API. By default it 
+assumes the use of [Jaeger](https://www.jaegertracing.io/). To install a simple development version of Jaeger, which 
+includes all components, use the following template (replace `fission` with the namespace where Fission Workflows is 
+installed):
+
+```bash
+kubectl -n fission create -f https://raw.githubusercontent.com/jaegertracing/jaeger-kubernetes/master/all-in-one/jaeger-all-in-one-template.yml
+```
+
+To view the Jaeger GUI navigate to the `jaeger-query` service. An example of a multi-task workflow execution:
+
+![Jaeger Tracing example](./assets/jaeger-example.png)

build/Dockerfile

@@ -19,4 +19,5 @@ RUN if ! $NOBUILD ; then \
 FROM scratch
 
 COPY --from=builder /go/src/github.com/fission/fission-workflows/fission-workflows-bundle /fission-workflows-bundle
+COPY --from=builder /go/src/github.com/fission/fission-workflows/fission-workflows-proxy /fission-workflows-proxy
 COPY --from=builder /go/src/github.com/fission/fission-workflows/fission-workflows /fission-workflows

build/build-linux.sh

@@ -2,4 +2,5 @@
 
 $(dirname $0)/build.sh --os linux --arch amd64 \
   --output-bundle "fission-workflows-bundle" \
-  --output-cli "fission-workflows"
+  --output-cli "fission-workflows" \
+  --output-proxy "fission-workflows-proxy"

build/build-osx.sh

@@ -2,4 +2,5 @@
 
 $(dirname $0)/build.sh --os darwin --arch 386 \
   --output-bundle "fission-workflows-bundle-osx" \
-  --output-cli "fission-workflows-osx"
+  --output-cli "fission-workflows-osx" \
+  --output-proxy "fission-workflows-proxy-osx"

build/build-windows.sh

@@ -2,4 +2,5 @@
 
 $(dirname $0)/build.sh --os windows --arch 386 \
   --output-bundle "fission-workflows-bundle-windows" \
-  --output-cli "fission-workflows-windows"
+  --output-cli "fission-workflows-windows" \
+  --output-proxy "fission-workflows-proxy-windows"

build/build.sh

@@ -38,6 +38,10 @@ while [[ $# -gt 0 ]]; do
       output_cli=$2
       shift
       ;;
+    --output-cli)
+      output_proxy=$2
+      shift
+      ;;
     -h| --help)
       echo "usage: build.sh [-c commit] [-d date] [-v version] [--arch arch] [--os os] [--output-bundle name] [--output-cli name]"
       exit 0;
@@ -61,6 +65,7 @@ goos=${goos:-linux}
 goarch=${goarch:-amd64}
 output_cli=${output_cli:-fission-workflows}
 output_bundle=${output_bundle:-fission-workflows-bundle}
+output_proxy=${output_proxy:-fission-workflows-proxy}
 
 echo "-------- Build config --------"
 echo "version: ${version}"
@@ -70,20 +75,29 @@ echo "goarch: ${goarch}"
 echo "goos: ${goos}"
 echo "output-cli: ${output_cli}"
 echo "output-bundle: ${output_bundle}"
+echo "output-proxy: ${output_proxy}"
 echo "------------------------------"
 
-# Build client
-CGO_ENABLED=0 GOOS=${goos} GOARCH=${goarch} go build \
-  -gcflags=-trimpath=${GOPATH} -asmflags=-trimpath=${GOPATH}\
-  -ldflags '-X "${versionPath}.buildDate=${date}"'\
-  -o ${output_cli}\
-  github.com/fission/fission-workflows/cmd/fission-workflows/
-echo "$(pwd)/${output_cli}"
-
 # Build bundle
 CGO_ENABLED=0 GOOS=${goos} GOARCH=${goarch} go build\
   -gcflags=-trimpath=${GOPATH} -asmflags=-trimpath=${GOPATH}\
   -ldflags '-X "${versionPath}.buildDate=${date}"'\
   -o ${output_bundle}\
   github.com/fission/fission-workflows/cmd/fission-workflows-bundle/
-echo "$(pwd)/${output_bundle}"
+echo "$(pwd)/${output_bundle}"
+
+# Build proxy
+CGO_ENABLED=0 GOOS=${goos} GOARCH=${goarch} go build\
+  -gcflags=-trimpath=${GOPATH} -asmflags=-trimpath=${GOPATH}\
+  -ldflags '-X "${versionPath}.buildDate=${date}"'\
+  -o ${output_proxy}\
+  github.com/fission/fission-workflows/cmd/fission-workflows-proxy/
+echo "$(pwd)/${output_proxy}"
+
+# Build client
+CGO_ENABLED=0 GOOS=${goos} GOARCH=${goarch} go build \
+  -gcflags=-trimpath=${GOPATH} -asmflags=-trimpath=${GOPATH}\
+  -ldflags '-X "${versionPath}.buildDate=${date}"'\
+  -o ${output_cli}\
+  github.com/fission/fission-workflows/cmd/fission-workflows/
+echo "$(pwd)/${output_cli}"

build/docker.sh

@@ -13,7 +13,7 @@ NOBUILD=${3:-false}
 # Build bundle images
 bundleImage=${IMAGE_REPO}/fission-workflows-bundle
 pushd ${BUILD_ROOT}/..
-if $NOBUILD ; then
+if ${NOBUILD} ; then
     echo "Using pre-build binaries..."
     if [ ! -f ./fission-workflows-bundle ]; then
         echo "Executable './fission-workflows-bundle' not found!"
@@ -33,21 +33,23 @@ docker build --tag="${bundleImage}:${IMAGE_TAG}" -f ${BUILD_ROOT}/Dockerfile \
 popd
 
 # Build bundle-dependent images
-echo "Building Fission runtime env..."
+echo "Building ${IMAGE_REPO}/workflow-env..."
 docker build --tag="${IMAGE_REPO}/workflow-env:${IMAGE_TAG}" ${BUILD_ROOT}/runtime-env/ \
     --no-cache \
     --build-arg BUNDLE_IMAGE=${bundleImage} \
     --build-arg BUNDLE_TAG=${IMAGE_TAG}
-echo "Building Fission build env..."
+echo "Building ${IMAGE_REPO}/workflow-build-env..."
 docker build --tag="${IMAGE_REPO}/workflow-build-env:${IMAGE_TAG}" ${BUILD_ROOT}/build-env/ \
     --no-cache \
     --build-arg BUNDLE_IMAGE=${bundleImage} \
     --build-arg BUNDLE_TAG=${IMAGE_TAG}
-echo "Building fission-workflows..."
+echo "Building ${IMAGE_REPO}/fission-workflows-cli..."
 docker build --tag="${IMAGE_REPO}/fission-workflows-cli:${IMAGE_TAG}" ${BUILD_ROOT}/cli/ \
     --no-cache \
     --build-arg BUNDLE_IMAGE=${bundleImage} \
     --build-arg BUNDLE_TAG=${IMAGE_TAG}
-
-# Remove intermediate images
-# docker rmi $(docker images -f "dangling=true" -q)
+echo "Building ${IMAGE_REPO}/fission workflows-proxy..."
+docker build --tag="${IMAGE_REPO}/workflows-proxy:${IMAGE_TAG}" ${BUILD_ROOT}/proxy/ \
+    --no-cache \
+    --build-arg BUNDLE_IMAGE=${bundleImage} \
+    --build-arg BUNDLE_TAG=${IMAGE_TAG}

build/proxy/Dockerfile

@@ -0,0 +1,10 @@
+ARG BUNDLE_IMAGE=fission-workflows-bundle
+ARG BUNDLE_TAG=latest
+FROM $BUNDLE_IMAGE:$BUNDLE_TAG as workflows-bundle
+
+FROM scratch
+
+COPY --from=workflows-bundle /fission-workflows-proxy /fission-workflows-proxy
+
+ENTRYPOINT ["/fission-workflows-proxy"]
+CMD ["-h"]

charts/fission-workflows/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 name: fission-workflows
-version: 0.5.0
-appVersion: 0.5.0
+version: 0.6.0
+appVersion: 0.6.0
 description: Fission Workflows is a fast workflow engine for serverless functions on Kubernetes
 keywords:
 - fission