feat: add istio config and odh overlay (#64)

* feat: add istio config and odh overlay * feat: initial version of istio resources and crd changes * feat: added tls and non-tls istio samples, refactored and cleaned up templates and operator * feat: add support for optional port number in gateway authority header * fix: added istio-system namespace for make certificates * feat: Updated README with better instructions, and for using Istio samples * Fix comment in modelregistry_types.go Co-authored-by: Wen Zhou <[email protected]> * fix: regenerate updated manifests after crd changes * chore: use descriptive import aliases * feat: remove odh istio config since it will be created in opendatahub-operator instead * fix: remove non-essential gateway properties from CRD * fix: added ns to certificates/clean target * fix: add wait for sample db start * fix: bump initialDelaySeconds for model registry deployment to avoid too many restarts * fix: add missing operator rbac permissions * fix: fix rbac role generation * fix: role needs to be created before being referenced in openshift * feat: fixes RHOAIENG-4218, added standard app labels for all k8s resources * feat: fixes RHOAIENG-4218, added role annotations for display-name and description * fix: add ENABLE_WEBHOOKS=false by default for 'make runr' * chore: clean up README instructions for using Istio samples * fix: make istioIngress optional * chore: added instructions in README to enabled OpenShift Service Mesh gateway route creation --------- Co-authored-by: Wen Zhou <[email protected]>
opendatahub-io · Apr 4, 2024 · 935f24f · 935f24f
1 parent c28bb0c
commit 935f24f
Show file tree

Hide file tree

Showing 42 changed files with 1,635 additions and 49 deletions.
diff --git a/.gitignore b/.gitignore
@@ -24,3 +24,6 @@ Dockerfile.cross
 *.swp
 *.swo
 *~
+
+# gateway TLS certificates
+certs/**
diff --git a/Makefile b/Makefile
@@ -1,6 +1,9 @@
 # Include images as env variables
 include ./config/overlays/odh/params.env
 
+# Include istio config as env variables
+include ./config/samples/istio/components/istio.env
+
 # Image URL to use all building/pushing image targets
 IMG_REGISTRY ?= "quay.io"
 IMG_ORG ?= "opendatahub"
@@ -13,6 +16,9 @@ ENVTEST_K8S_VERSION = 1.28.0
 # Kustomize overlay to use for deploy/undeploy
 OVERLAY ?= "default"
 
+# Disable operator webhooks by default for local testing
+ENABLE_WEBHOOKS ?= false
+
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(shell go env GOPATH)/bin
@@ -90,7 +96,7 @@ build: sync-images manifests generate fmt vet ## Build manager binary.
 
 .PHONY: run
 run: manifests generate fmt vet ## Run a controller from your host.
-	go run ./cmd/main.go
+	ENABLE_WEBHOOKS=$(ENABLE_WEBHOOKS) go run ./cmd/main.go
 
 # If you wish to build the manager image targeting other platforms you can use the --platform flag.
 # (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it.
@@ -184,3 +190,25 @@ $(CONTROLLER_GEN): $(LOCALBIN)
 envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.
 $(ENVTEST): $(LOCALBIN)
 	test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@$(ENVTEST_VERSION)
+
+.PHONY: certificates
+certificates:
+	# generate TLS certs
+	scripts/generate_certs.sh $(DOMAIN)
+	# create secrets from TLS certs
+	$(KUBECTL) create secret -n istio-system generic modelregistry-sample-rest-credential \
+      --from-file=tls.key=certs/modelregistry-sample-rest.domain.key \
+      --from-file=tls.crt=certs/modelregistry-sample-rest.domain.crt \
+      --from-file=ca.crt=certs/domain.crt
+	$(KUBECTL) create secret -n istio-system generic modelregistry-sample-grpc-credential \
+      --from-file=tls.key=certs/modelregistry-sample-grpc.domain.key \
+      --from-file=tls.crt=certs/modelregistry-sample-grpc.domain.crt \
+      --from-file=ca.crt=certs/domain.crt
+
+.PHONY: certificates/clean
+certificates/clean:
+	# delete cert files
+	mkdir -p certs
+	rm -f certs/*
+	# delete k8s secrets
+	$(KUBECTL) delete -n istio-system secrets modelregistry-sample-rest-credential modelregistry-sample-grpc-credential
diff --git a/README.md b/README.md
@@ -11,33 +11,193 @@ The controller reconciles `ModelRegistry` Custom Resources to create a service f
 You’ll need a Kubernetes cluster to run against. You can use [KIND](https://sigs.k8s.io/kind) to get a local cluster for testing, or run against a remote cluster.
 **Note:** Your controller will automatically use the current context in your kubeconfig file (i.e. whatever cluster `kubectl cluster-info` shows).
 
-`ModelRegistry` service needs a PostgreSQL or MySQL database. A sample Postgres configuration for testing is included 
+`ModelRegistry` service needs a PostgreSQL or MySQL database. A sample Postgres configuration for testing (without TLS security) is included 
 in [postgres-db.yaml](config/samples/postgres/postgres-db.yaml) and a sample MySQL configuration for testing is included
 in [mysql-db.yaml](config/samples/mysql/mysql-db.yaml).
 To use another PostgreSQL instance, comment the line that includes the sample DB in [kustomization.yaml](config/samples/postgres/kustomization.yaml) and to use 
-your own mysql instance comment the line that includes the sample DB in [kustomization.yaml](config/samples/mysql/kustomization.yaml). 
+your own mysql instance comment the line that includes the sample DB in [kustomization.yaml](config/samples/mysql/kustomization.yaml).
 
-### Running on the cluster
-1. Install Instances of Custom Resources using one of the two database options:
+The operator supports creating model registries using [Istio](https://istio.io/) for security including [Authorino](https://github.com/Kuadrant/authorino) for authorization. 
+It also supports [Istio Gateways](https://istio.io/latest/docs/concepts/traffic-management/#gateways) for exposing service endpoints for clients outside the Istio service mesh.
 
-For MySQL use the command:
+### Istio Configuration
+_Skip this section if you are not using Istio._
 
-```sh
-kubectl apply -k config/samples/mysql
+*NOTE* that this section describes how to configure a couple of files in the kustomize samples in this project. However, there are only a handful of extra configuration properties needed in an Istio model registry. 
+
+For using the Istio based samples, both Istio and Authorino MUST be installed before deploying the operator. 
+An Authorino instance MUST also be configured as a [custom authz](https://istio.io/latest/docs/tasks/security/authorization/authz-custom/) provider in the Istio control plane. 
+
+Both Istio and Authorino along with the authz provider can be easily enabled in the Open Data Hub data science cluster instance. 
+
+If Istio or Authorino is installed in the cluster after deploying this controller, restart the controller by deleting the pod `model-registry-operator-controller-manager` in `model-registry-operator-system` namespace. 
+If Model Registry component has been installed as an Open Data Hub operator component, the operator namespace will be `opendatahub`. 
+
+If Authorino provider is from a non Open Data Hub cluster, configure its selector labels in the [authconfig-labels.yaml](config/samples/istio/components/authconfig-labels.yaml) file. 
+
+To use the Istio model registry samples the following configuration data is needed in the [istio.env](config/samples/istio/components/istio.env) file:
+
+* AUTH_PROVIDER - name of the authorino external auth provider configured in the Istio control plane (defaults to `opendatahub-auth-provider` for Open Data Hub data science cluster with OpenShift Service Mesh enabled).
+* DOMAIN - hostname domain suffix for gateway endpoints. 
+This depends upon your cluster's external load balancer config. In OpenShift clusters, it can be obtained with the command:
+```shell
+oc get ingresses.config/cluster -o jsonpath='{.spec.domain}'
 ```
-For PostgreSQL use the command:
+* ISTIO_INGRESS - name of the Istio Ingress Gateway (defaults to `ingressgateway`). 
+* REST_CREDENTIAL_NAME - Kubernetes secret in IngressGateway namespace (typically `istio-system`) containing TLS certificates for REST service (defaults to `modelregistry-sample-rest-credential`). 
+* GRPC_CREDENTIAL_NAME - Kubernetes secret in IngressGateway namespace containing TLS certificates for gRPC service (defaults to `modelregistry-sample-grpc-credential`).
+
+### Running on the cluster
+1. Deploy the controller to the cluster using the `latest` docker image:
 
 ```sh
+make deploy
+```
+
+2. The operator includes multiple samples that use [kustomize](https://kustomize.io/) to create a sample model registry `modelregistry-sample`.
+
+* [MySQL without Istio](config/samples/mysql) plain Kubernetes model registry services with a sample MySQL database
+* [PostgreSQL without Istio](config/samples/postgres) plain Kubernetes model registry services with a sample PostgreSQL database
+* [MySQL with Istio](config/samples/istio/mysql) MySQL database, Istio, and plaintext Gateway
+* [PostgreSQL with Istio](config/samples/istio/postgres) PostgreSQL database, Istio, and plaintext Gateway
+* [MySQL with Istio and TLS](config/samples/istio/mysql-tls) MySQL database, Istio, and TLS Gateway endpoints
+* [PostgreSQL with Istio and TLS](config/samples/istio/postgres-tls) PostgreSQL database, Istio, and TLS Gateway endpoints
+
+#### Istio Samples
+**WARNING:** Istio samples without TLS are only meant for testing and demos to avoid having to create TLS certificates. They should only be used in local development clusters. 
+
+##### Authorization
+For all Istio samples, a Kubernetes user or serviceaccount authorization token MUST be passed in calls to model registry services using the header:
+
+```http request
+Authorization: Bearer sha256~xxx
+```
+
+In OpenShift clusters, the user session token can be obtained using the command:
+
+```shell
+oc whoami -t
+```
+
+##### TLS Certificates
+The project [Makefile](Makefile) includes targets to manage test TLS certificates using a self signed CA certificate. 
+To create test certificates in the directory [certs](certs) and Kubernetes secrets in the `istio-system` namespace, use the command:
+
+```shell
+make certificates
+```
+
+The test CA certificate is generated in the file [certs/domain.crt](certs/domain.crt) along with other certificates. See [generate_certs.sh](scripts/generate_certs.sh) for details. 
+
+To cleanup the certificates and Kubernetes secrets, use the command:
+
+```shell
+make certificates/clean
+```
+
+To disable Istio Gateway creation, create a kustomize overlay that removes the `gateway` yaml section in model registry custom resource or manually edit a sample yaml and it's corresponding `replacements.yaml` helper. 
+
+##### Enable Namespace Istio Injection
+If using upstream Istio (i.e. not OpenShift Service Mesh), enable Istio proxy injection in your test namespace by using the command:
+
+```shell
+kubectl label namespace <namespace> istio-injection=enabled --overwrite
+```
+
+If using OpenShift Service Mesh, enable it by adding the namespace to the control plane (e.g. ODH Istio control plane `data-science-smcp` below) by using the command:
+
+```shell
+kubectl apply -f -<<EOF
+apiVersion: maistra.io/v1
+kind: ServiceMeshMember
+metadata:
+  name: default
+spec:
+  controlPlaneRef:
+    name: data-science-smcp
+    namespace: istio-system
+EOF
+```
+
+##### Enable OpenShift Service Mesh Route Creation
+If using OpenShift Service Mesh with Open Data Hub Operator and Gateway Route creation is disabled, it can be enabled by using the command:
+
+```shell
+kubectl patch smcp data-science-smcp -n istio-system --type='json' -p='[{"op": "replace", "path": "/spec/gateways/openshiftRoute/enabled", "value": true}]'
+```
+
+3. For Istio samples, first configure properties in [istio.env](config/samples/istio/components/istio.env). 
+Install a model registry instance using **ONE** of the following commands:
+
+```shell
+kubectl apply -k config/samples/mysql
 kubectl apply -k config/samples/postgres
+kubectl apply -k config/samples/istio/mysql
+kubectl apply -k config/samples/istio/postgres
+kubectl apply -k config/samples/istio/mysql-tls
+kubectl apply -k config/samples/istio/postgres-tls
+```
+
+This will create the appropriate model registry resource, which will be reconciled in the controller to create a model registry deployment with other Kubernetes, Istio, and Authorino resources as needed.
+
+4. Check that the sample model registry was created using the command:
+
+```shell
+kubectl describe mr modelregistry-sample
+```
+
+Check the `Status` of the model registry resource for failed Conditions. 
+
+For Istio Gateway examples, consult your Istio configuration to verify gateway endpoint creation. For OpenShift Service Mesh with gateway route creation enabled, look for model registry routes using the command:
+
+```shell
+kubectl get route -n istio-system
+```
+
+To verify the REST gateway service, use the following command:
+
+```shell
+curl -H "Authorization: Bearer $TOKEN" --cacert certs/domain.crt https://modelregistry-sample-rest.$DOMAIN/api/model_registry/v1alpha3/registered_models
+```
+
+Where, `$TOKEN` and `$DOMAIN` environment variables are set to the client token and host domain. If using OpenShift, the token and domain can be set using the commands:
+
+```shell
+export TOKEN=`oc whoami -t`
+export DOMAIN=`oc get ingresses.config/cluster -o jsonpath='{.spec.domain}'`
+```
+
+If using a non-TLS gateway, use the command:
+
+```shell
+curl -H "Authorization: Bearer $TOKEN" http://modelregistry-sample-rest.$DOMAIN/api/model_registry/v1alpha3/registered_models
+```
+
+The output should be a list of all registered models in the registry, e.g. for an empty registry:
+
+```json
+{"items":[],"nextPageToken":"","pageSize":0,"size":0}
+```
+
+5. To delete the sample model registry, use **ONE** of the following commands based on the sample type deployed earlier:
+
+```shell
+kubectl delete -k config/samples/mysql
+kubectl delete -k config/samples/postgres
+kubectl delete -k config/samples/istio/mysql
+kubectl delete -k config/samples/istio/postgres
+kubectl delete -k config/samples/istio/mysql-tls
+kubectl delete -k config/samples/istio/postgres-tls
 ```
 
-2. Build and push your image to the location specified by `IMG`:
+### Building local docker image for development
+1. Build and push your image to the location specified by `IMG`:
 
 ```sh
 make docker-build docker-push IMG=<some-registry>/model-registry-operator:tag
 ```
 
-3. Deploy the controller to the cluster with the image specified by `IMG`:
+2. Deploy the controller to the cluster with the image specified by `IMG`:
 
 ```sh
 make deploy IMG=<some-registry>/model-registry-operator:tag

diff --git a/api/v1alpha1/modelregistry_types.go b/api/v1alpha1/modelregistry_types.go
@@ -173,6 +173,113 @@ type GrpcSpec struct {
 	Image string `json:"image,omitempty"`
 }
 
+type TLSServerSettings struct {
+
+	//+kubebuilder:default=SIMPLE
+	//+kubebuilder:validation:Enum=SIMPLE;MUTUAL;ISTIO_MUTUAL;OPTIONAL_MUTUAL
+
+	// The value of this field determines how TLS is enforced.
+	// SIMPLE: Secure connections with standard TLS semantics. In this mode client certificate is not requested during handshake.
+	//
+	// MUTUAL: Secure connections to the downstream using mutual TLS by presenting server certificates for authentication. A client certificate will also be requested during the handshake and at least one valid certificate is required to be sent by the client.
+	//
+	// ISTIO_MUTUAL: Secure connections from the downstream using mutual TLS by presenting server certificates for authentication. Compared to Mutual mode, this mode uses certificates, representing gateway workload identity, generated automatically by Istio for mTLS authentication. When this mode is used, all other TLS fields should be empty.
+	//
+	// OPTIONAL_MUTUAL: Similar to MUTUAL mode, except that the client certificate is optional. Unlike SIMPLE mode, A client certificate will still be explicitly requested during handshake, but the client is not required to send a certificate. If a client certificate is presented, it will be validated. ca_certificates should be specified for validating client certificates.
+	Mode string `json:"mode"`
+
+	// The name of the secret that holds the TLS certs including the CA certificates.
+	// An Opaque secret should contain the following
+	// keys and values: `tls.key: <privateKey>` and `tls.crt: <serverCert>` or
+	// `key: <privateKey>` and `cert: <serverCert>`.
+	// For mutual TLS, `cacert: <CACertificate>` and `crl: <CertificateRevocationList>`
+	// can be provided in the same secret or a separate secret named `<secret>-cacert`.
+	// A TLS secret for server certificates with an additional `tls.ocsp-staple` key
+	// for specifying OCSP staple information, `ca.crt` key for CA certificates
+	// and `ca.crl` for certificate revocation list is also supported.
+	// Only one of server certificates and CA certificate
+	// or credentialName can be specified.
+	//+optional
+	CredentialName *string `json:"credentialName,omitempty"`
+}
+
+type ServerConfig struct {
+	//+kubebuilder:validation:Minimum=0
+	//+kubebuilder:validation:Maximum=65535
+
+	// Listen port for server connections, defaults to 80 without TLS and 443 when TLS settings are present.
+	Port *int32 `json:"port,omitempty"`
+
+	// Set of TLS related options that govern the server's behavior. Use
+	// these options to control if all http requests should be redirected to
+	// https, and the TLS modes to use.
+	//+optional
+	TLS *TLSServerSettings `json:"tls,omitempty"`
+}
+
+type GatewayConfig struct {
+	//+kubebuilder:required
+
+	// Domain name for Gateway configuration
+	Domain string `json:"domain"`
+
+	//+kubebuilder:default=ingressgateway
+
+	// Value of label `istio` used to identify the Ingress Gateway
+	IstioIngress *string `json:"istioIngress,omitempty"`
+
+	// Maistra/OpenShift Servicemesh control plane name
+	//+optional
+	ControlPlane *string `json:"controlPlane,omitempty"`
+
+	// Rest gateway server config
+	Rest ServerConfig `json:"rest"`
+
+	// gRPC  gateway server config
+	Grpc ServerConfig `json:"grpc"`
+}
+
+type IstioConfig struct {
+	//+kubebuilder:required
+
+	// Authorino authentication provider name
+	AuthProvider string `json:"authProvider"`
+
+	// Authorino AuthConfig selector labels
+	//+optional
+	AuthConfigLabels map[string]string `json:"authConfigLabels,omitempty"`
+
+	//+kubebuilder:required
+	//+kubebuilder:default=ISTIO_MUTUAL
+	//+kubebuilder:Enum=DISABLE;SIMPLE;MUTUAL;ISTIO_MUTUAL
+
+	// DestinationRule TLS mode. Defaults to ISTIO_MUTUAL.
+	//
+	// DISABLE: Do not setup a TLS connection to the upstream endpoint.
+	//
+	// SIMPLE: Originate a TLS connection to the upstream endpoint.
+	//
+	// MUTUAL: Secure connections to the upstream using mutual TLS by presenting
+	// client certificates for authentication.
+	//
+	// ISTIO_MUTUAL: Secure connections to the upstream using mutual TLS by presenting
+	// client certificates for authentication.
+	// Compared to Mutual mode, this mode uses certificates generated
+	// automatically by Istio for mTLS authentication. When this mode is
+	// used, all other fields in `ClientTLSSettings` should be empty.
+	TlsMode string `json:"tlsMode,omitempty"`
+
+	// Optional Istio Gateway for registry services.
+	// Gateway is not created if set to null (default).
+	//+optional
+	Gateway *GatewayConfig `json:"gateway,omitempty"`
+
+	// Optional Authorino AuthConfig credential audiences. This depends on the cluster identity provider.
+	// If not specified, operator will determine the cluster's audience using its own service account.
+	//+optional
+	Audiences []string `json:"audiences,omitempty"`
+}
+
 // ModelRegistrySpec defines the desired state of ModelRegistry.
 // One of `postgres` or `mysql` database configurations MUST be provided!
 type ModelRegistrySpec struct {
@@ -184,7 +291,7 @@ type ModelRegistrySpec struct {
 	// Configuration for REST endpoint
 	Rest RestSpec `json:"rest"`
 
-	//+kubebuilder: required
+	//+kubebuilder:required
 
 	// Configuration for gRPC endpoint
 	Grpc GrpcSpec `json:"grpc"`
@@ -207,6 +314,10 @@ type ModelRegistrySpec struct {
 	// initialization (Optional Parameter)
 	//+optional
 	DowngradeDbSchemaVersion *int64 `json:"downgrade_db_schema_version,omitempty"`
+
+	// Istio servicemesh configuration options
+	//+optional
+	Istio *IstioConfig `json:"istio,omitempty"`
 }
 
 // ModelRegistryStatus defines the observed state of ModelRegistry
-Original file line number
+Diff line change
@@ Expand Up / @@ -24,3 +24,6 @@ Dockerfile.cross @@
     *.swp
     *.swo
     *~
+    # gateway TLS certificates
+    certs/**