Added support for ephemeral volumes and ingress creation support

.buildkite/test-rayjob-sample-yamls.yml

@@ -0,0 +1,71 @@
+- label: Test RayJob Sample YAMLs
+  instance_size: large
+  image: golang:1.19
+  commands:
+    # Install Go
+    - export PATH=$PATH:/usr/local/go/bin
+
+    # Install kind
+    - curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64
+    - chmod +x ./kind
+    - mv ./kind /usr/local/bin/kind
+
+    # Install Docker
+    - bash scripts/install-docker.sh
+
+    # Delete dangling clusters
+    - kind delete clusters --all
+
+    # Install kubectl.
+    - curl -LO https://dl.k8s.io/release/v1.27.3/bin/linux/amd64/kubectl
+    - curl -LO "https://dl.k8s.io/release/v1.27.3/bin/linux/amd64/kubectl.sha256"
+    - echo "$(cat kubectl.sha256)  kubectl" | sha256sum --check
+    - install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+
+    # Create the cluster
+    - time kind create cluster --wait 120s --config tests/framework/config/kind-config-buildkite.yml
+    - docker ps
+
+    # Now the kind node is running, it exposes port 6443 in the dind-daemon network.
+    - kubectl config set clusters.kind-kind.server https://docker:6443
+
+    # Verify that kubectl works
+    - kubectl version
+    - kubectl cluster-info
+    - kubectl get nodes
+    - kubectl get pods --all-namespaces
+
+    # Install Helm
+    - curl -Lo helm.tar.gz https://get.helm.sh/helm-v3.12.2-linux-amd64.tar.gz
+    - tar -zxvf helm.tar.gz
+    - mv linux-amd64/helm /usr/local/bin/helm
+    - helm repo add kuberay https://ray-project.github.io/kuberay-helm/
+    - helm repo update
+
+    # Install KubeRay operator
+    - pushd ray-operator
+    - IMG=kuberay/operator:nightly make docker-image
+    - kind load docker-image kuberay/operator:nightly
+    - popd
+
+    - pushd helm-chart/kuberay-operator
+    # Use helm --wait instead of kubectl wait to prevent flakiness. See #618
+    - helm install kuberay-operator --set image.repository=kuberay/operator --set image.tag=nightly --wait --timeout=5m0s .
+    - popd
+
+    - echo "Kuberay operator successfully installed."
+
+    # Delete kind clusters
+    - kind delete clusters --all
+
+    # Install python 3.10 and pip
+    - apt-get update
+    - apt-get install -y python3.11 python3.11-venv
+    - python3 -m venv .venv
+    - source .venv/bin/activate
+
+    # Install requirements
+    - pip install -r tests/framework/config/requirements.txt
+
+    # Run test
+    - BUILDKITE_ENV=true python3 tests/test_sample_rayjob_yamls.py

apiserver/DEVELOPMENT.md

@@ -102,7 +102,9 @@ make install
 #To use the helm charts
 make undeploy
 
+
 #To use the configuration
+
 make uninstall
 ```
 
@@ -122,8 +124,10 @@ When developing and testing with kind you might want to execute these targets to
 
 ```bash
 #To create a new API server image and to deploy it on a new cluster
+
 make docker-image cluster load-image deploy
 
+
 #To create a new API server image, operator image and deploy them on a new cluster
 make docker-image operator-image cluster load-image load-operator-image deploy deploy-operator
 ```

apiserver/pkg/model/converter.go

@@ -107,6 +107,10 @@ func FromCrdToApiCluster(cluster *v1alpha1.RayCluster, events []v1.Event) *api.C
 		ClusterState: string(cluster.Status.State),
 	}
 
+	if len(cluster.ObjectMeta.Annotations) > 0 {
+		pbCluster.Annotations = cluster.ObjectMeta.Annotations
+	}
+
 	// loop container and find the resource
 	pbCluster.ClusterSpec = PopulateRayClusterSpec(cluster.Spec)
 
@@ -146,6 +150,7 @@ func PopulateHeadNodeSpec(spec v1alpha1.HeadGroupSpec) *api.HeadGroupSpec {
 		ServiceType:     string(spec.ServiceType),
 		Image:           spec.Template.Annotations[util.RayClusterImageAnnotationKey],
 		ComputeTemplate: spec.Template.Annotations[util.RayClusterComputeTemplateAnnotationKey],
+		Volumes:         PopulateVolumes(&spec.Template),
 	}
 
 	for _, annotation := range getNodeDefaultAnnotations() {
@@ -162,6 +167,10 @@ func PopulateHeadNodeSpec(spec v1alpha1.HeadGroupSpec) *api.HeadGroupSpec {
 		headNodeSpec.Labels = spec.Template.Labels
 	}
 
+	if *spec.EnableIngress {
+		headNodeSpec.EnableIngress = true
+	}
+
 	// Here we update environment only for a container named 'ray-head'
 	if container, _, ok := util.GetContainerByName(spec.Template.Spec.Containers, "ray-head"); ok && len(container.Env) > 0 {
 		env := make(map[string]string)
@@ -196,6 +205,7 @@ func PopulateWorkerNodeSpec(specs []v1alpha1.WorkerGroupSpec) []*api.WorkerGroup
 			GroupName:       spec.GroupName,
 			Image:           spec.Template.Annotations[util.RayClusterImageAnnotationKey],
 			ComputeTemplate: spec.Template.Annotations[util.RayClusterComputeTemplateAnnotationKey],
+			Volumes:         PopulateVolumes(&spec.Template),
 		}
 
 		for _, annotation := range getNodeDefaultAnnotations() {

apiserver/pkg/model/converter_test.go

@@ -11,7 +11,7 @@ import (
 )
 
 var (
-	enableIngress          = false
+	enableIngress          = true
 	headNodeReplicas int32 = 1
 	workerReplicas   int32 = 5
 )
@@ -195,6 +195,22 @@ var workerSpecTest = v1alpha1.WorkerGroupSpec{
 	},
 }
 
+var ClusterSpecTest = v1alpha1.RayCluster{
+	ObjectMeta: metav1.ObjectMeta{
+		Name:      "raycluster-sample",
+		Namespace: "default",
+		Annotations: map[string]string{
+			"kubernetes.io/ingress.class": "nginx",
+		},
+	},
+	Spec: v1alpha1.RayClusterSpec{
+		HeadGroupSpec: headSpecTest,
+		WorkerGroupSpecs: []v1alpha1.WorkerGroupSpec{
+			workerSpecTest,
+		},
+	},
+}
+
 var expectedAnnotations = map[string]string{
 	"custom": "value",
 }
@@ -220,6 +236,9 @@ func TestPopulateHeadNodeSpec(t *testing.T) {
 	if groupSpec.ServiceAccount != "account" {
 		t.Errorf("failed to convert service account")
 	}
+	if groupSpec.EnableIngress != *headSpecTest.EnableIngress {
+		t.Errorf("failed to convert enableIngress")
+	}
 	if groupSpec.ImagePullSecret != "foo" {
 		t.Errorf("failed to convert image pull secret")
 	}
@@ -254,6 +273,13 @@ func TestPopulateWorkerNodeSpec(t *testing.T) {
 	}
 }
 
+func TestPopulateRayClusterSpec(t *testing.T) {
+	cluster := FromCrdToApiCluster(&ClusterSpecTest, []v1.Event{})
+	if len(cluster.Annotations) != 1 {
+		t.Errorf("failed to convert cluster's annotations")
+	}
+}
+
 func TestPopulateTemplate(t *testing.T) {
 	template := FromKubeToAPIComputeTemplate(&configMapWithoutTolerations)
 	if len(template.Tolerations) != 0 {

apiserver/pkg/model/volumes.go

@@ -0,0 +1,103 @@
+package model
+
+import (
+	api "github.com/ray-project/kuberay/proto/go_client"
+	v1 "k8s.io/api/core/v1"
+)
+
+func PopulateVolumes(podTemplate *v1.PodTemplateSpec) []*api.Volume {
+	if len(podTemplate.Spec.Volumes) == 0 {
+		return nil
+	}
+	var volumes []*api.Volume
+	for _, vol := range podTemplate.Spec.Volumes {
+		mount := GetVolumeMount(podTemplate, vol.Name)
+		if vol.VolumeSource.HostPath != nil {
+			// Host Path
+			volumes = append(volumes, &api.Volume{
+				Name:                 vol.Name,
+				MountPath:            mount.MountPath,
+				Source:               vol.VolumeSource.HostPath.Path,
+				MountPropagationMode: GetVolumeMountPropagation(mount),
+				VolumeType:           api.Volume_VolumeType(api.Volume_HOSTTOCONTAINER),
+				HostPathType:         GetVolumeHostPathType(&vol),
+			})
+			continue
+
+		}
+		if vol.VolumeSource.PersistentVolumeClaim != nil {
+			// PVC
+			volumes = append(volumes, &api.Volume{
+				Name:                 vol.Name,
+				MountPath:            mount.MountPath,
+				MountPropagationMode: GetVolumeMountPropagation(mount),
+				VolumeType:           api.Volume_PERSISTENT_VOLUME_CLAIM,
+				ReadOnly:             vol.VolumeSource.PersistentVolumeClaim.ReadOnly,
+			})
+			continue
+		}
+		if vol.VolumeSource.Ephemeral != nil {
+			// Ephimeral
+			request := vol.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec.Resources.Requests[v1.ResourceStorage]
+			volume := api.Volume{
+				Name:                 vol.Name,
+				MountPath:            mount.MountPath,
+				MountPropagationMode: GetVolumeMountPropagation(mount),
+				VolumeType:           api.Volume_EPHEMERAL,
+				AccessMode:           GetAccessMode(&vol),
+				Storage:              request.String(),
+			}
+			if vol.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec.StorageClassName != nil {
+				volume.StorageClassName = *vol.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec.StorageClassName
+			}
+			volumes = append(volumes, &volume)
+			continue
+		}
+	}
+	return volumes
+}
+
+func GetVolumeMount(podTemplate *v1.PodTemplateSpec, vol string) *v1.VolumeMount {
+	for _, container := range podTemplate.Spec.Containers {
+		for _, mount := range container.VolumeMounts {
+			if mount.Name == vol {
+				return &mount
+			}
+		}
+	}
+	return nil
+}
+
+func GetVolumeMountPropagation(mount *v1.VolumeMount) api.Volume_MountPropagationMode {
+	if mount.MountPropagation == nil {
+		return api.Volume_NONE
+	}
+	if *mount.MountPropagation == v1.MountPropagationHostToContainer {
+		return api.Volume_HOSTTOCONTAINER
+	}
+	if *mount.MountPropagation == v1.MountPropagationBidirectional {
+		return api.Volume_BIDIRECTIONAL
+	}
+	return api.Volume_NONE
+}
+
+func GetVolumeHostPathType(vol *v1.Volume) api.Volume_HostPathType {
+	if *vol.VolumeSource.HostPath.Type == v1.HostPathFile {
+		return api.Volume_FILE
+	}
+	return api.Volume_DIRECTORY
+}
+
+func GetAccessMode(vol *v1.Volume) api.Volume_AccessMode {
+	modes := vol.VolumeSource.Ephemeral.VolumeClaimTemplate.Spec.AccessModes
+	if len(modes) == 0 {
+		return api.Volume_RWO
+	}
+	if modes[0] == v1.ReadOnlyMany {
+		return api.Volume_ROX
+	}
+	if modes[0] == v1.ReadWriteMany {
+		return api.Volume_RWX
+	}
+	return api.Volume_RWO
+}