ray-project · kevin85421 · Aug 13, 2023 · Jul 24, 2023 · Aug 7, 2023 · Aug 7, 2023
diff --git a/apiserver/DEVELOPMENT.md b/apiserver/DEVELOPMENT.md
@@ -4,70 +4,118 @@ This guide covers the purpose, requirements, and deployment of the Kuberay API S
 
 ## Requirements
 
-| Software | Version  |                                                            Link |
-| :------- | :------: | ------------------------------------------------------------: |
+| Software | Version  |                                                                Link |
+| :------- | :------: | ------------------------------------------------------------------: |
 | kubectl  | v1.18.3+ | [Download](https://kubernetes.io/docs/tasks/tools/install-kubectl/) |
-| Go       |  v1.13+  |                            [Download](https://golang.org/dl/) |
-| Docker   |  19.03+  |                  [Download](https://docs.docker.com/install/) |
+| Go       |  v1.17   |                                  [Download](https://golang.org/dl/) |
+| Docker   |  19.03+  |                        [Download](https://docs.docker.com/install/) |
+| GNU Make |  3.81+   |                                                                     |
+| curl     |  7.88+   |                                                                     |
+| helm     | v3.12.1  |                      [Install](https://helm.sh/docs/intro/install/) |
+
+### Optional Development Tools
+
+These tools are downloaded and installed when they are needed. The directory of the download is `../bin`.
+Typing `make dev-tools` will download and install all of them. The `make clean-dev-tools` command can be used to remove all the tools from the filesystem.
+
+| Software      | Version  |                                                                    Link |
+| :-------      | :------: | -----------------------------------------------------------------------:|
+| kind          | v0.19.0  | [Install](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) |
+| golangci-lint | v1.50.1  | [Install](https://golangci-lint.run/usage/install/)                     |
+| kustomize     | v3.8.7   | [install](https://kubectl.docs.kubernetes.io/installation/kustomize/)   |
+| gofumpt       | v0.3.1   | To install `go install mvdan.cc/[email protected]`                         |
+| goimports     | latest   | To install `go install golang.org/x/tools/cmd/goimports@latest`         |
 
 ## Purpose
 
 The Kuberay API Server is designed to simplify the lifecycle management of Ray clusters for users who may not be well-versed in Kubernetes. It provides a RESTful web service to manage Ray cluster Kubernetes resources.
 
 ## Build and Deployment
 
-The backend service can be deployed locally or within a Kubernetes cluster. The HTTP service listens on port 8888.
+The backend service can be deployed locally or within a Kubernetes cluster. The HTTP service listens on port 8888, the RPC port on 8887.
 
 ### Pre-requisites
 
-Ensure that the admin Kubernetes configuration file is located at `~/.kube/config`.
+Ensure that the admin Kubernetes configuration file is located at `~/.kube/config`. As a convenience, there are two makefile targets provided to help you manage a local kind cluster:
 
-### Local Deployment
+* `make cluster` -- creates a 3 node cluster (1 control plane 2 worker) named ray-api-server-cluster
+* `make clean-cluster` -- deletes the cluster created with the `cluster` target
+
+### Local Development
 
 #### Build
 
 ```bash
-go build -a -o raymgr cmd/main.go
+#To build the executable
+make build
+
+#To start the executable build above
+../bin/kuberay-apiserver
 ```
 
-#### Start Service
+#### Test
 
 ```bash
-./raymgr
+#To run the unit tests
+make test
+```
+
+#### Start Local Service
+
+This will start the api server on your development machine. The golang race detector is turned on when starting the api server this way. It will use Kubernetes configuration file located at `~/.kube/config`. The service will not start if you do not have a connection to a Kubernetes cluster.
+
+```bash
+make run
 ```
 
 #### Access
 
-Access the service at `localhost:8888`.
+Access the service at `localhost:8888` for http, and `locahost:8887` for the RPC port.
 
 ### Kubernetes Deployment
 
-#### Build
+#### Build Image
 
 ```bash
-./docker-image-builder.sh
+#creates an image with the tag kuberay/apiserver:latest
+make docker-image 
 ```
 
-This script will build and optionally push the image to the remote Docker Hub (hub.byted.org).
-
-#### Start Service
+#### Start Kubernetes Deployment
 
 ```bash
-kubectl apply -f deploy/
-```
+#Optionally, to load the api server image into the local kind cluster created with make cluster
+make load-image
 
-#### Access
+#To use the  helm charts
+make deploy
 
-To obtain the port, run the following command:
-
-```bash
-NODE_PORT=$(kubectl get -o jsonpath="{.spec.ports[0].nodePort}" services backend-service -n ray-system)
+#To use the configuration from deploy/base
+make install
 ```
 
-To obtain the node, run the following command:
+#### Stop Kubernetes Deployment
 
 ```bash
-NODE_IP=$(kubectl get nodes -o jsonpath='{ $.items[*].status.addresses[?(@.type=="InternalIP")].address }')
+#To use the  helm charts
+make undeploy
+
+#To use the configuration 
+make uninstall
 ```
 
-Select any IP address from the output, and use `NODE_IP:NODE_PORT` to access the service.
+#### Local Kind Cluster Deployment
+
+As a convenience for local development the following `make` targets are provided:
+
+* `make cluster` -- creates a local kind cluster, using the configuration from `hack/kind-cluster-config.yaml`. It creates a port mapping allowing for the service running in the kind cluster to be accessed on  `localhost:318888` for HTTP and `localhost:318887` for RPC.
+* `make clean-cluster` -- deletes the local kind cluster created with `make cluster`
+* `load-image` -- loads the docker image defined by the `IMG` make variable into the kind cluster. The default value for variable is: `kuberay/apiserver:latest`. The name of the image can be changed by using `make load-image -e IMG=<your image name and tag>`
+* `operator-image` -- Build the operator image to be loaded in your kind cluster. The tag for the operator image is `kuberay/operator:latest`. This step is optional.
+* `load-operator-image` -- Load the operator image to the kind cluster created with `create-kind-cluster`. The tag for the operator image is `kuberay/operator:latest`, and the tag can be overridden using `make load-operator-image -E OPERATOR_IMAGE_TAG=<operator tag>`. To use the nightly operator tag, set the tag to `nightly`.
+* `deploy-operator` -- Deploy operator into your cluster.  The tag for the operator image is `kuberay/operator:latest`.
+* `undeploy-operator` -- Undeploy operator from your cluster
+
+#### Access API Server in the Cluster
+
+Access the service at `localhost:318888` for http and `locahost:318887` for the RPC port.
diff --git a/apiserver/Makefile b/apiserver/Makefile
@@ -0,0 +1,195 @@
+BUILD_TIME      := $(shell date "+%F %T")
+COMMIT_SHA1     := $(shell git rev-parse HEAD )
+REPO_ROOT_BIN	:= $(shell dirname ${PWD})/bin
+
+# Image URL to use all building/pushing image targets
+IMG_TAG ?=latest
+IMG ?= kuberay/apiserver:$(IMG_TAG)
+
+# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
+ifeq (,$(shell go env GOBIN))
+GOBIN=$(shell go env GOPATH)/bin
+else
+GOBIN=$(shell go env GOBIN)
+endif
+
+# Setting SHELL to bash allows bash commands to be executed by recipes.
+# This is a requirement for 'setup-envtest.sh' in the test target.
+# Options are set to exit when a recipe line exits non-zero or a piped command fails.
+SHELL = /usr/bin/env bash -o pipefail
+.SHELLFLAGS = -ec
+
+# Container Engine to be used for building images
+ENGINE ?= docker
+
+all: build
+
+##@ General
+
+# The help target prints out all targets with their descriptions organized
+# beneath their categories. The categories are represented by '##@' and the
+# target descriptions by '##'. The awk commands is responsible for reading the
+# entire set of makefiles included in this invocation, looking for lines of the
+# file as xyz: ## something, and then pretty-format the target and help. Then,
+# if there's a line with ##@ something, that gets pretty-printed as a category.
+# More info on the usage of ANSI control characters for terminal formatting:
+# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
+# More info on the awk command:
+# http://linuxcommand.org/lc3_adv_awk.php
+
+help: ## Display this help.
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+
+##@ Development
+
+fmt: ## Run go fmt against code.
+	go fmt ./...
+
+vet: ## Run go vet against code.
+	go vet ./...
+
+fumpt: gofumpt ## Run gofmtumpt against code.
+	$(GOFUMPT) -l -w .
+
+imports: goimports ## Run goimports against code.
+	$(GOIMPORTS) -l -w .	
+
+test: fmt vet fumpt imports lint  ## Run unit tests.
+	go test ./... -race -coverprofile ray-kube-api-server-coverage.out
+
+lint: golangci-lint fmt vet fumpt imports ## Run the linter.
+	$(GOLANGCI_LINT) run  --timeout=3m	
+
+##@ Build
+
+COMMIT := $(shell git rev-parse --short HEAD)
+VERSION := $(shell git describe --tags $(shell git rev-list --tags --max-count=1))
+DATE=$(shell date -u +'%Y-%m-%dT%H:%M:%SZ')
+REPO=github.com/ray-project/kuberay
+
+BUILD_FLAGS ?= -ldflags="-X '${REPO}/apiserver/cmd/version.Version=$(VERSION)' \
+	-X '${REPO}/apiserver/cmd/version.gitCommit=$(COMMIT)' \
+	-X '${REPO}/apiserver/cmd/version.buildDate=$(DATE)'"
+
+build: fmt vet fumpt imports lint ## Build api server binary.
+	go build $(BUILD_FLAGS) -o ${REPO_ROOT_BIN}/kuberay-apiserver cmd/main.go
+
+run: fmt vet fumpt imports lint ## Run the api server from your host.
+	go run -race cmd/main.go
+
+docker-image: test ## Build image with the api server.
+	${ENGINE} build -t ${IMG} -f Dockerfile ..
+
+docker-push: ## Push image with the api server.
+	${ENGINE} push ${IMG}
+
+##@ Deployment
+.PHONY: install
+install: kustomize ## Install the kuberay api server to the K8s cluster specified in ~/.kube/config.
+	cd deploy/base && $(KUSTOMIZE) edit set image kuberay/apiserver=$(IMG)
+	($(KUSTOMIZE) build deploy/base | kubectl create -f -) || $(KUSTOMIZE) build deploy/base | kubectl replace -f -
+
+.PHONY: uninstall
+uninstall: ## Install the kuberay api server to the K8s cluster specified in ~/.kube/config.
+	$(KUSTOMIZE) build deploy/base | kubectl delete -f -
+
+.PHONY: deploy
+deploy: ## Deploy via helm the kuberay api server to the K8s cluster specified in ~/.kube/config.
+	helm upgrade --install kuberay-apiserver ../helm-chart/kuberay-apiserver --wait \
+	--set image.tag=${IMG_TAG} --set image.pullPolicy=IfNotPresent
+
+.PHONY: undeploy
+undeploy: ## Undeploy via helm the kuberay api server to the K8s cluster specified in ~/.kube/config.
+	helm delete kuberay-apiserver --wait
+
+##@ Development Tools
+
+## Location to install dependencies to
+$(REPO_ROOT_BIN):
+	mkdir -p $(REPO_ROOT_BIN)
+
+## Tool Binaries
+KUSTOMIZE ?= $(REPO_ROOT_BIN)/kustomize
+GOIMPORTS ?= $(REPO_ROOT_BIN)/goimports
+GOFUMPT ?= $(REPO_ROOT_BIN)/gofumpt
+GOLANGCI_LINT ?= $(REPO_ROOT_BIN)/golangci-lint
+KIND ?= $(REPO_ROOT_BIN)/kind
+
+
+## Tool Versions
+KUSTOMIZE_VERSION ?= v3.8.7
+GOFUMPT_VERSION ?= v0.3.1
+GOIMPORTS_VERSION ?= latest
+GOLANGCI_LINT_VERSION ?= v1.50.1
+KIND_VERSION ?= v0.19.0
+
+KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"
+.PHONY: kustomize
+kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary.
+$(KUSTOMIZE): $(REPO_ROOT_BIN)
+	test -s $(KUSTOMIZE) || (curl -Ss $(KUSTOMIZE_INSTALL_SCRIPT) | bash -s -- $(subst v,,$(KUSTOMIZE_VERSION)) $(REPO_ROOT_BIN);)
+
+.PHONY: goimports
+goimports: $(GOIMPORTS) ## Download goimports locally if necessary
+$(GOIMPORTS): $(REPO_ROOT_BIN)
+	test -s $(GOIMPORTS) || GOBIN=$(REPO_ROOT_BIN) go install golang.org/x/tools/cmd/goimports@$(GOIMPORTS_VERSION)
+
+.PHONY: gofumpt
+gofumpt: $(GOFUMPT) ## Download gofumpt locally if necessary.
+$(GOFUMPT): $(REPO_ROOT_BIN)	
+	test -s $(GOFUMPT) || GOBIN=$(REPO_ROOT_BIN) go install mvdan.cc/gofumpt@$(GOFUMPT_VERSION)
+
+.PHONY: golangci-lint
+golangci-lint: $(GOLANGCI_LINT) ## Download golangci_lint locally if necessary.
+$(GOLANGCI_LINT): $(REPO_ROOT_BIN)
+	test -s $(GOLANGCI_LINT) || (curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | bash -s -- -b $(REPO_ROOT_BIN)/ $(GOLANGCI_LINT_VERSION))
+
+.PHONY: kind
+kind: $(KIND) ## Download kind locally if necessary.
+$(KIND): $(REPO_ROOT_BIN)	
+	test -s $(KIND) || GOBIN=$(REPO_ROOT_BIN) go install sigs.k8s.io/kind@$(KIND_VERSION)
+
+.PHONY: dev-tools
+dev-tools: kind golangci-lint gofumpt kustomize goimports ## Install all development tools
+
+.PHONY: clean-dev-tools
+clean-dev-tools: ## Remove all development tools
+	rm -f $(REPO_ROOT_BIN)/golangci-lint
+	rm -f $(REPO_ROOT_BIN)/gofumpt
+	rm -f $(REPO_ROOT_BIN)/kustomize
+	rm -f $(REPO_ROOT_BIN)/goimports
+	rm -f $(REPO_ROOT_BIN)/kind
+
+
+##@ Testing Setup and Tools
+KIND_CONFIG ?= hack/kind-cluster-config.yaml
+KIND_CLUSTER_NAME ?= ray-api-server-cluster
+OPERATOR_IMAGE_TAG ?= latest
+.PHONY: cluster
+cluster: kind ## Start kind development cluster.
+	$(KIND) create cluster -n $(KIND_CLUSTER_NAME) --config $(KIND_CONFIG)
+
+.PHONY: clean-cluster
+clean-cluster: kind ## Delete kind development cluster.
+	$(KIND) delete cluster -n $(KIND_CLUSTER_NAME)
+
+.PHONY: load-image
+load-image: ## Load the api server image to the kind cluster created with create-kind-cluster.
+	$(KIND) load docker-image $(IMG) -n $(KIND_CLUSTER_NAME)
+
+.PHONY: operator-image
+operator-image: ## Build the operator image to be loaded in your kind cluster.
+	cd ../ray-operator && $(MAKE) docker-image -e IMG=kuberay/operator:$(OPERATOR_IMAGE_TAG)
+
+.PHONY: deploy-operator
+deploy-operator: ## Deploy operator via helm into the K8s cluster specified in ~/.kube/config.
+	helm upgrade --install raycluster ../helm-chart/kuberay-operator --wait \
+	--set image.tag=${OPERATOR_IMAGE_TAG} --set image.pullPolicy=IfNotPresent
+
+.PHONY: undeploy-operator
+undeploy-operator: ## Undeploy operator via helm from the K8s cluster specified in ~/.kube/config.
+	helm delete -n raycluster-operator raycluster --wait
+
+.PHONY: load-operator-image
+load-operator-image: ## Load the operator image to the kind cluster created with create-kind-cluster.
+	$(KIND) load docker-image kuberay/operator:$(OPERATOR_IMAGE_TAG) -n $(KIND_CLUSTER_NAME)
diff --git a/apiserver/deploy/base/apiserver.yaml b/apiserver/deploy/base/apiserver.yaml
@@ -118,3 +118,8 @@ rules:
   verbs:
   - get
   - list
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ray-system
diff --git a/apiserver/deploy/base/kustomization.yaml b/apiserver/deploy/base/kustomization.yaml
@@ -1,11 +1,21 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-
 namespace: ray-system
-
 resources:
 - apiserver.yaml
-
+patchesJson6902:
+- patch: |-
+    - op: replace
+      path: /spec/template/spec/containers/0/imagePullPolicy
+      value: IfNotPresent
+  target:
+    kind: Deployment
+    name: kuberay-apiserver
+    version: v1
 commonLabels:
-  app.kubernetes.io/name: kuberay
   app.kubernetes.io/component: kuberay-apiserver
+  app.kubernetes.io/name: kuberay
+images:
+- name: kuberay/apiserver
+  newName: kuberay/apiserver
+  newTag: latest