diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..e2cdc09c --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,25 @@ +{ + "name": "Kubebuilder DevContainer", + "image": "golang:1.22", + "features": { + "ghcr.io/devcontainers/features/docker-in-docker:2": {}, + "ghcr.io/devcontainers/features/git:1": {} + }, + + "runArgs": ["--network=host"], + + "customizations": { + "vscode": { + "settings": { + "terminal.integrated.shell.linux": "/bin/bash" + }, + "extensions": [ + "ms-kubernetes-tools.vscode-kubernetes-tools", + "ms-azuretools.vscode-docker" + ] + } + }, + + "onCreateCommand": "bash .devcontainer/post-install.sh" +} + diff --git a/.devcontainer/post-install.sh b/.devcontainer/post-install.sh new file mode 100644 index 00000000..265c43ee --- /dev/null +++ b/.devcontainer/post-install.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -x + +curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64 +chmod +x ./kind +mv ./kind /usr/local/bin/kind + +curl -L -o kubebuilder https://go.kubebuilder.io/dl/latest/linux/amd64 +chmod +x kubebuilder +mv kubebuilder /usr/local/bin/ + +KUBECTL_VERSION=$(curl -L -s https://dl.k8s.io/release/stable.txt) +curl -LO "https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl" +chmod +x kubectl +mv kubectl /usr/local/bin/kubectl + +docker network create -d=bridge --subnet=172.19.0.0/24 kind + +kind version +kubebuilder version +docker --version +go version +kubectl version --client diff --git a/.dockerignore b/.dockerignore index 0f046820..a3aab7af 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,3 @@ # More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file # Ignore build and test binaries. bin/ -testbin/ diff --git a/.github/workflows/auto-add-issues-to-project.yaml b/.github/workflows/auto-add-issues-to-project.yaml deleted file mode 100644 index e7dd1781..00000000 --- a/.github/workflows/auto-add-issues-to-project.yaml +++ /dev/null @@ -1,28 +0,0 @@ -name: Auto Add Issues to Tracking boards -on: - issues: - types: - - opened -jobs: - add-to-project: - name: Add issue to projects - runs-on: ubuntu-latest - steps: - - name: Generate github-app token - id: app-token - uses: getsentry/action-github-app-token@v2 - with: - app_id: ${{ secrets.DEVOPS_APP_ID }} - private_key: ${{ secrets.DEVOPS_APP_PRIVATE_KEY }} - - uses: actions/add-to-project@v0.5.0 - with: - project-url: https://github.com/orgs/opendatahub-io/projects/40 - github-token: ${{ steps.app-token.outputs.token }} - - uses: actions/add-to-project@v0.5.0 - with: - project-url: https://github.com/orgs/opendatahub-io/projects/45 - github-token: ${{ steps.app-token.outputs.token }} - - uses: actions/add-to-project@v0.5.0 - with: - project-url: https://github.com/orgs/opendatahub-io/projects/42 - github-token: ${{ steps.app-token.outputs.token }} \ No newline at end of file diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..f40d3657 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,23 @@ +name: Lint + +on: + push: + pull_request: + +jobs: + lint: + name: Run on Ubuntu + runs-on: ubuntu-latest + steps: + - name: Clone the code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '~1.22' + + - name: Run linter + uses: golangci/golangci-lint-action@v6 + with: + version: v1.61 diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml new file mode 100644 index 00000000..2fbee952 --- /dev/null +++ b/.github/workflows/test-e2e.yml @@ -0,0 +1,36 @@ +name: E2E Tests + +on: +# push: +# pull_request: + workflow_dispatch: + +jobs: + test-e2e: + name: Run on Ubuntu + runs-on: ubuntu-latest + steps: + - name: Clone the code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '~1.22' + + - name: Install the latest version of kind + run: | + curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64 + chmod +x ./kind + sudo mv ./kind /usr/local/bin/kind + + - name: Verify kind installation + run: kind version + + - name: Create kind cluster + run: kind create cluster + + - name: Running Test e2e + run: | + go mod tidy + make test-e2e diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..d08d90cb --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,24 @@ +name: Tests + +on: + push: + pull_request: + +jobs: + test: + name: Run on Ubuntu + runs-on: ubuntu-latest + steps: + - name: Clone the code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '~1.22' + + - name: Running Tests + run: | + go mod tidy + git diff --exit-code || exit 1 + make test diff --git a/.gitignore b/.gitignore index a25b34c1..63874787 100644 --- a/.gitignore +++ b/.gitignore @@ -1,26 +1,31 @@ - # Binaries for programs and plugins *.exe *.exe~ *.dll *.so *.dylib -bin -testbin/* +bin/* +Dockerfile.cross -# Test binary, build with `go test -c` +# Test binary, built with `go test -c` *.test # Output of the go coverage tool, specifically when used with LiteIDE *.out -# Kubernetes Generated files - skip generated files, except for vendored files +# Go workspace file +go.work +# Kubernetes Generated files - skip generated files, except for vendored files !vendor/**/zz_generated.* # editor and IDE paraphernalia .idea +.vscode *.swp *.swo *~ .vscode + +# Other +.envrc diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 00000000..f0ce26ec --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,49 @@ +run: + timeout: 5m + allow-parallel-runners: true + +issues: + # don't skip warning about doc comments + # don't exclude the default set of lint + exclude-use-default: false + # restore some of the defaults + # (fill in the rest as needed) + exclude-rules: + - path: "api/*" + linters: + - lll + - path: "internal/*" + linters: + - dupl + - lll +linters: + disable-all: true + enable: + - dupl + - errcheck + - copyloopvar + - ginkgolinter + - goconst + - gocyclo + - gofmt + - goimports + - gosimple + - govet + - ineffassign + - lll + - misspell + - nakedret + - prealloc + - revive + - staticcheck + - typecheck + - unconvert + - unparam + - unused + +linters-settings: + revive: + rules: + - name: comment-spacings + goconst: + ignore-strings: "^true$" diff --git a/Containerfile b/Containerfile index 0fd64dfa..1ef08699 100644 --- a/Containerfile +++ b/Containerfile @@ -1,5 +1,7 @@ # Build the manager binary -FROM registry.access.redhat.com/ubi9/go-toolset:1.21 as builder +FROM registry.access.redhat.com/ubi9/go-toolset:1.22.7 as builder +ARG TARGETOS +ARG TARGETARCH WORKDIR /workspace # Copy the Go Modules manifests @@ -10,16 +12,19 @@ COPY go.sum go.sum RUN go mod download # Copy the go source -COPY main.go main.go +COPY cmd/main.go cmd/main.go COPY api/ api/ -COPY controllers/ controllers/ +COPY internal/ internal/ # Build +# the GOARCH has not a default value to allow the binary be built according to the host where the command +# was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO +# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, +# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. USER root -RUN CGO_ENABLED=0 GOOS=linux go build -a -o manager main.go +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} go build -a -o manager cmd/main.go # Use distroless as minimal base image to package the manager binary -# Refer to https://github.com/GoogleContainerTools/distroless for more details FROM registry.access.redhat.com/ubi8/ubi-minimal:8.6 WORKDIR / COPY --from=builder /workspace/manager . diff --git a/Makefile b/Makefile index 33585b93..471979ae 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,23 @@ - # Image URL to use all building/pushing image targets IMG ?= quay.io/${USER}/odh-model-controller:latest # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. -ENVTEST_K8S_VERSION = 1.29 +ENVTEST_K8S_VERSION = 1.31.0 + +# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) +ifeq (,$(shell go env GOBIN)) +GOBIN=$(shell go env GOPATH)/bin +else +GOBIN=$(shell go env GOBIN) +endif -ENGINE ?= docker +# CONTAINER_TOOL defines the container tool to be used for building images. +# Be aware that the target commands are only tested with Docker which is +# scaffolded by default. However, you might want to replace it to use other +# tools. (i.e. podman) +ENGINE ?= docker # Backwards compatibility +CONTAINER_TOOL ?= $(ENGINE) # Setting SHELL to bash allows bash commands to be executed by recipes. -# This is a requirement for 'setup-envtest.sh' in the test target. # Options are set to exit when a recipe line exits non-zero or a piped command fails. SHELL = /usr/bin/env bash -o pipefail .SHELLFLAGS = -ec @@ -19,7 +29,7 @@ all: build # The help target prints out all targets with their descriptions organized # beneath their categories. The categories are represented by '##@' and the -# target descriptions by '##'. The awk commands is responsible for reading the +# target descriptions by '##'. The awk command is responsible for reading the # entire set of makefiles included in this invocation, looking for lines of the # file as xyz: ## something, and then pretty-format the target and help. Then, # if there's a line with ##@ something, that gets pretty-printed as a category. @@ -36,15 +46,16 @@ help: ## Display this help. .PHONY: manifests manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. - # Any customization needed, apply to the webhook_patch.yaml file + # Any customization needed, apply to a patch in the kustomize.yaml file on webhooks $(CONTROLLER_GEN) rbac:roleName=odh-model-controller-role,headerFile="hack/manifests_boilerplate.yaml.txt" crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases -external-manifests: +# TODO: Evaluate if this is still needed +external-manifests: go get github.com/kserve/modelmesh-serving $(CONTROLLER_GEN) crd \ paths=${GOPATH}/pkg/mod/github.com/kserve/modelmesh-serving@v0.8.0/apis/serving/v1alpha1 \ output:crd:artifacts:config=config/crd/external - + go get github.com/openshift/api $(CONTROLLER_GEN) crd \ paths=${GOPATH}/pkg/mod/github.com/openshift/api@v3.9.0+incompatible/route/v1 \ @@ -69,30 +80,77 @@ vet: ## Run go vet against code. .PHONY: test test: manifests generate fmt vet envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" POD_NAMESPACE=default \ - MESH_NAMESPACE=istio-system CONTROL_PLANE_NAME=istio-system go test -v ./controllers/... -coverprofile cover.out - -.PHONY: e2e-test -e2e-test: manifests generate fmt vet ## Run e2e-tests. - POD_NAMESPACE=default go test ./test/e2e/... + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" POD_NAMESPACE=default \ + MESH_NAMESPACE=istio-system CONTROL_PLANE_NAME=istio-system go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out + +# TODO(user): To use a different cluster than Kind for e2e tests, modify the setup under 'tests/e2e'. +# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. +# Prometheus and CertManager are installed by default; skip with: +# - PROMETHEUS_INSTALL_SKIP=true +# - CERT_MANAGER_INSTALL_SKIP=true +.PHONY: test-e2e +test-e2e: manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind. + @command -v kind >/dev/null 2>&1 || { \ + echo "Kind is not installed. Please install Kind manually."; \ + exit 1; \ + } + @kind get clusters | grep -q 'kind' || { \ + echo "No Kind cluster is running. Please start a Kind cluster before running the e2e tests."; \ + exit 1; \ + } + POD_NAMESPACE=default go test ./test/e2e/ -v -ginkgo.v + +.PHONY: lint +lint: golangci-lint ## Run golangci-lint linter + $(GOLANGCI_LINT) run + +.PHONY: lint-fix +lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes + $(GOLANGCI_LINT) run --fix ##@ Build .PHONY: build -build: generate fmt vet ## Build manager binary. - go build -o bin/manager main.go +build: manifests generate fmt vet ## Build manager binary. + go build -o bin/manager cmd/main.go .PHONY: run run: manifests generate fmt vet ## Run a controller from your host. - go run ./main.go + go run ./cmd/main.go +# If you wish to build the manager image targeting other platforms you can use the --platform flag. +# (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it. +# More info: https://docs.docker.com/develop/develop-images/build_enhancements/ .PHONY: container-build -container-build: test ## Build docker image with the manager. - ${ENGINE} build . -f ./Containerfile -t ${IMG} +container-build: ## Build docker image with the manager. + $(CONTAINER_TOOL) build -t ${IMG} -f ./Containerfile . .PHONY: container-push container-push: ## Push docker image with the manager. - ${ENGINE} push ${IMG} + $(CONTAINER_TOOL) push ${IMG} + +# PLATFORMS defines the target platforms for the manager image be built to provide support to multiple +# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: +# - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/ +# - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/ +# - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=> then the export will fail) +# To adequately provide solutions that are compatible with multiple platforms, you should consider using this option. +PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le +.PHONY: docker-buildx +docker-buildx: ## Build and push docker image for the manager for cross-platform support + # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile + sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross + - $(CONTAINER_TOOL) buildx create --name odh-model-controller-builder + $(CONTAINER_TOOL) buildx use odh-model-controller-builder + - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross . + - $(CONTAINER_TOOL) buildx rm odh-model-controller-builder + rm Dockerfile.cross + +.PHONY: build-installer +build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. + mkdir -p dist + cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + $(KUSTOMIZE) build config/default > dist/install.yaml ##@ Deployment @@ -102,31 +160,22 @@ endif .PHONY: install install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/crd | kubectl apply -f - + $(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f - .PHONY: uninstall uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - + $(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - .PHONY: deploy deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default | kubectl apply -f - + $(KUSTOMIZE) build config/default | $(KUBECTL) apply -f - .PHONY: undeploy -undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f - - -.PHONY: deploy-dev -deploy-dev: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/overlays/dev | kubectl apply -f - - -.PHONY: undeploy-dev -undeploy-dev: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/overlays/dev | kubectl delete --ignore-not-found=$(ignore-not-found) -f - +undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - -##@ Build Dependencies +##@ Dependencies ## Location to install dependencies to LOCALBIN ?= $(shell pwd)/bin @@ -134,27 +183,50 @@ $(LOCALBIN): mkdir -p $(LOCALBIN) ## Tool Binaries +KUBECTL ?= kubectl KUSTOMIZE ?= $(LOCALBIN)/kustomize CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen ENVTEST ?= $(LOCALBIN)/setup-envtest +GOLANGCI_LINT = $(LOCALBIN)/golangci-lint ## Tool Versions -KUSTOMIZE_VERSION ?= v3.8.7 -CONTROLLER_TOOLS_VERSION ?= v0.14.0 +KUSTOMIZE_VERSION ?= v5.5.0 +CONTROLLER_TOOLS_VERSION ?= v0.16.4 +ENVTEST_VERSION ?= release-0.19 +GOLANGCI_LINT_VERSION ?= v1.61.0 -KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" .PHONY: kustomize kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. $(KUSTOMIZE): $(LOCALBIN) - curl -s $(KUSTOMIZE_INSTALL_SCRIPT) | bash -s -- $(subst v,,$(KUSTOMIZE_VERSION)) $(LOCALBIN) + $(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v5,$(KUSTOMIZE_VERSION)) .PHONY: controller-gen controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. $(CONTROLLER_GEN): $(LOCALBIN) - GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) + $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen,$(CONTROLLER_TOOLS_VERSION)) -ENVTEST_PACKAGE_VERSION = v0.0.0-20240320141353-395cfc7486e6 .PHONY: envtest -envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. +envtest: $(ENVTEST) ## Download setup-envtest locally if necessary. $(ENVTEST): $(LOCALBIN) - GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@$(ENVTEST_PACKAGE_VERSION) + $(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION)) + +.PHONY: golangci-lint +golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. +$(GOLANGCI_LINT): $(LOCALBIN) + $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION)) + +# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist +# $1 - target path with name of binary +# $2 - package url which can be installed +# $3 - specific version of package +define go-install-tool +@[ -f "$(1)-$(3)" ] || { \ +set -e; \ +package=$(2)@$(3) ;\ +echo "Downloading $${package}" ;\ +rm -f $(1) || true ;\ +GOBIN=$(LOCALBIN) go install $${package} ;\ +mv $(1) $(1)-$(3) ;\ +} ;\ +ln -sf $(1)-$(3) $(1) +endef diff --git a/PROJECT b/PROJECT index b9deeda5..63560ddb 100644 --- a/PROJECT +++ b/PROJECT @@ -1,13 +1,18 @@ +# Code generated by tool. DO NOT EDIT. +# This file is used to track the info used to scaffold your project +# and allow the plugins properly work. +# More info: https://book.kubebuilder.io/reference/project-config.html domain: opendatahub.io layout: -- go.kubebuilder.io/v3 +- go.kubebuilder.io/v4 +multigroup: true projectName: odh-model-controller repo: github.com/opendatahub-io/odh-model-controller -version: "3" resources: - api: crdVersion: v1 namespaced: true + controller: true domain: opendatahub.io group: nim kind: Account @@ -15,4 +20,43 @@ resources: version: v1 webhooks: validation: true - webhookVersion: v1 \ No newline at end of file + webhookVersion: v1 +- controller: true + domain: kserve.io + external: true + group: serving + kind: InferenceService + path: github.com/kserve/kserve/pkg/apis/serving/v1beta1 + version: v1beta1 + webhooks: + validation: true + webhookVersion: v1 +- controller: true + core: true + group: core + kind: ConfigMap + path: k8s.io/api/core/v1 + version: v1 +- controller: true + domain: kserve.io + external: true + group: serving + kind: ServingRuntime + path: github.com/kserve/kserve/pkg/apis/serving/v1alpha1 + version: v1alpha1 +- controller: true + core: true + group: core + kind: Secret + path: k8s.io/api/core/v1 + version: v1 +- domain: knative.dev + external: true + group: serving + kind: Service + path: knative.dev/serving/pkg/apis/serving/v1 + version: v1 + webhooks: + validation: true + webhookVersion: v1 +version: "3" diff --git a/README.md b/README.md index 9823edfb..a5c84837 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ -# ODH Model Controller +# odh-model-controller +// TODO(user): Add simple overview of use/purpose +## Description The controller will watch the Predictor custom resource events to extend the KServe modelmesh-serving controller behavior with the following capabilities: @@ -9,37 +11,94 @@ capabilities: It has been developed using **Golang** and **[Kubebuilder](https://book.kubebuilder.io/quick-start.html)**. -## Implementation detail +## Getting Started +### Prerequisites +- go version v1.22.0+ +- podman version v5.2+. +- kubectl version v1.11.3+. +- Access to a Kubernetes v1.11.3+ cluster. +### To Deploy on the cluster +**Build and push your image to the location specified by `IMG`:** -## Developer docs +```sh +make container-build container-push IMG=/odh-model-controller:tag +``` + +**NOTE:** This image ought to be published in the personal registry you specified. +And it is required to have access to pull the image from the working environment. +Make sure you have the proper permission to the registry if the above commands don’t work. + +**Install the CRDs into the cluster:** + +```sh +make install +``` + +**Deploy the Manager to the cluster with the image specified by `IMG`:** -Follow the instructions below if you want to extend the controller -functionality: +```sh +make deploy IMG=/odh-model-controller:tag +``` + +> **NOTE**: If you encounter RBAC errors, you may need to grant yourself cluster-admin +privileges or be logged in as admin. + +**Create instances of your solution** +You can apply the samples (examples) from the config/sample: -### Run unit tests +```sh +kubectl apply -k config/samples/ +``` -Unit tests have been developed using the [**Kubernetes envtest -framework**](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/envtest). +>**NOTE**: Ensure that the samples has default values to test it out. -Run the following command to execute them: +### To Uninstall +**Delete the instances (CRs) from the cluster:** -```shell -make test +```sh +kubectl delete -k config/samples/ ``` -### Deploy local changes +**Delete the APIs(CRDs) from the cluster:** + +```sh +make uninstall +``` -Build a new image with your local changes and push it to `` (by -default `quay.io/${USER}/odh-model-controller:latest`). +**UnDeploy the controller from the cluster:** -```shell -make -e IMG= container-build container-push +```sh +make undeploy ``` -Deploy the manager using the image in your registry: +## Project Distribution + +Following are the steps to build the installer and distribute this project to users. + +1. Build the installer for the image built and published in the registry: + +```sh +make build-installer IMG=/odh-model-controller:tag +``` + +NOTE: The makefile target mentioned above generates an 'install.yaml' +file in the dist directory. This file contains all the resources built +with Kustomize, which are necessary to install this project without +its dependencies. + +2. Using the installer + +Users can just run kubectl apply -f to install the project, i.e.: + +```sh +kubectl apply -f https://raw.githubusercontent.com//odh-model-controller//dist/install.yaml +``` + +## Contributing +// TODO(user): Add detailed information on how you would like others to contribute to this project + +**NOTE:** Run `make help` for more information on all potential `make` targets -```shell -make deploy -e K8S_NAMESPACE= -e IMG= -``` \ No newline at end of file +More information can be found via the [Kubebuilder Documentation](https://book.kubebuilder.io/introduction.html) diff --git a/api/nim/v1/account_types.go b/api/nim/v1/account_types.go index bf18250f..06173e51 100644 --- a/api/nim/v1/account_types.go +++ b/api/nim/v1/account_types.go @@ -7,50 +7,48 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -type ( - // AccountSpec defines the desired state of an Account object. - AccountSpec struct { - // A reference to the Secret containing the NGC API Key. - APIKeySecret corev1.ObjectReference `json:"apiKeySecret"` - } - - // AccountStatus defines the observed state of an Account object. - AccountStatus struct { - // A reference to the Template for NIM ServingRuntime. - RuntimeTemplate *corev1.ObjectReference `json:"runtimeTemplate,omitempty"` - // A reference to the ConfigMap with data for NIM deployment. - NIMConfig *corev1.ObjectReference `json:"nimConfig,omitempty"` - // A reference to the Secret for pulling NIM images. - NIMPullSecret *corev1.ObjectReference `json:"nimPullSecret,omitempty"` - - Conditions []metav1.Condition `json:"conditions,omitempty"` - } - - // Account is used for adopting a NIM Account for Open Data Hub. - // - // +kubebuilder:object:root=true - // +kubebuilder:subresource:status - // - // +kubebuilder:printcolumn:name="Template",type="string",JSONPath=".status.runtimeTemplate.name",description="Template for ServingRuntime" - // +kubebuilder:printcolumn:name="ConfigMap",type="string",JSONPath=".status.nimConfig.name",description="ConfigMap of NIM data" - // +kubebuilder:printcolumn:name="Secret",type="string",JSONPath=".status.nimPullSecret.name",description="Secret for pulling NIM images" - Account struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec AccountSpec `json:"spec,omitempty"` - Status AccountStatus `json:"status,omitempty"` - } - - // AccountList is used for encapsulating Account items. - // - // +kubebuilder:object:root=true - AccountList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []Account `json:"items"` - } -) +// AccountSpec defines the desired state of an Account object. +type AccountSpec struct { + // A reference to the Secret containing the NGC API Key. + APIKeySecret corev1.ObjectReference `json:"apiKeySecret"` +} + +// AccountStatus defines the observed state of an Account object. +type AccountStatus struct { + // A reference to the Template for NIM ServingRuntime. + RuntimeTemplate *corev1.ObjectReference `json:"runtimeTemplate,omitempty"` + // A reference to the ConfigMap with data for NIM deployment. + NIMConfig *corev1.ObjectReference `json:"nimConfig,omitempty"` + // A reference to the Secret for pulling NIM images. + NIMPullSecret *corev1.ObjectReference `json:"nimPullSecret,omitempty"` + + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +// +kubebuilder:printcolumn:name="Template",type="string",JSONPath=".status.runtimeTemplate.name",description="Template for ServingRuntime" +// +kubebuilder:printcolumn:name="ConfigMap",type="string",JSONPath=".status.nimConfig.name",description="ConfigMap of NIM data" +// +kubebuilder:printcolumn:name="Secret",type="string",JSONPath=".status.nimPullSecret.name",description="Secret for pulling NIM images" + +// Account is used for adopting a NIM Account for Open Data Hub. +type Account struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec AccountSpec `json:"spec,omitempty"` + Status AccountStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// AccountList is used for encapsulating Account items. +type AccountList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Account `json:"items"` +} func init() { SchemeBuilder.Register(&Account{}, &AccountList{}) diff --git a/api/nim/v1/groupversion_info.go b/api/nim/v1/groupversion_info.go index 12cba59e..a40fe35d 100644 --- a/api/nim/v1/groupversion_info.go +++ b/api/nim/v1/groupversion_info.go @@ -1,5 +1,8 @@ // Copyright (c) 2024 Red Hat, Inc. +// Package v1 contains API Schema definitions for the nim v1 API group. +// +kubebuilder:object:generate=true +// +groupName=nim.opendatahub.io package v1 import ( @@ -7,12 +10,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/scheme" ) -// +groupName=nim.opendatahub.io -// +kubebuilder:object:generate=true -// +kubebuilder:validation:Required - var ( - GroupVersion = schema.GroupVersion{Group: "nim.opendatahub.io", Version: "v1"} + // GroupVersion is group version used to register these objects. + GroupVersion = schema.GroupVersion{Group: "nim.opendatahub.io", Version: "v1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme. SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} - Install = SchemeBuilder.AddToScheme + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme ) diff --git a/api/nim/v1/zz_generated.deepcopy.go b/api/nim/v1/zz_generated.deepcopy.go index fddd6a4e..fc63f07a 100644 --- a/api/nim/v1/zz_generated.deepcopy.go +++ b/api/nim/v1/zz_generated.deepcopy.go @@ -1,7 +1,7 @@ //go:build !ignore_autogenerated /* -Copyright 2022. +Copyright 2024. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cmd/main.go b/cmd/main.go new file mode 100644 index 00000000..e3ef8a5a --- /dev/null +++ b/cmd/main.go @@ -0,0 +1,291 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "crypto/tls" + "flag" + "os" + "slices" + "strconv" + + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. + _ "k8s.io/client-go/plugin/pkg/client/auth" + + istiov1beta1 "istio.io/client-go/pkg/apis/security/v1beta1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + corecontroller "github.com/opendatahub-io/odh-model-controller/internal/controller/core" + "github.com/opendatahub-io/odh-model-controller/internal/controller/nim" + servingcontroller "github.com/opendatahub-io/odh-model-controller/internal/controller/serving" + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" + webhooknimv1 "github.com/opendatahub-io/odh-model-controller/internal/webhook/nim/v1" + webhookservingv1 "github.com/opendatahub-io/odh-model-controller/internal/webhook/serving/v1" + webhookservingv1beta1 "github.com/opendatahub-io/odh-model-controller/internal/webhook/serving/v1beta1" + // +kubebuilder:scaffold:imports +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + utils.RegisterSchemes(scheme) +} + +func getEnvAsBool(name string, defaultValue bool) bool { + valStr := os.Getenv(name) + if val, err := strconv.ParseBool(valStr); err == nil { + return val + } + return defaultValue +} + +func main() { + var metricsAddr string + var enableLeaderElection bool + var probeAddr string + var secureMetrics bool + var enableHTTP2 bool + var tlsOpts []func(*tls.Config) + var monitoringNS string + var enableMRInferenceServiceReconcile bool + + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metrics endpoint binds to. "+ + "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.BoolVar(&enableLeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + flag.BoolVar(&secureMetrics, "metrics-secure", false, // TODO: restore to true by default. + "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.BoolVar(&enableHTTP2, "enable-http2", false, + "If set, HTTP/2 will be enabled for the metrics and webhook servers") + flag.StringVar(&monitoringNS, "monitoring-namespace", "", + "The Namespace where the monitoring stack's Prometheus resides.") + flag.BoolVar(&enableMRInferenceServiceReconcile, "model-registry-inference-reconcile", false, + "Enable model registry inference service reconciliation. ") + opts := zap.Options{ + Development: true, + } + opts.BindFlags(flag.CommandLine) + flag.Parse() + + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + // if the enable-http2 flag is false (the default), http/2 should be disabled + // due to its vulnerabilities. More specifically, disabling http/2 will + // prevent from being vulnerable to the HTTP/2 Stream Cancellation and + // Rapid Reset CVEs. For more information see: + // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 + // - https://github.com/advisories/GHSA-4374-p667-p6c8 + disableHTTP2 := func(c *tls.Config) { + setupLog.Info("disabling http/2") + c.NextProtos = []string{"http/1.1"} + } + + if !enableHTTP2 { + tlsOpts = append(tlsOpts, disableHTTP2) + } + + webhookServer := webhook.NewServer(webhook.Options{ + TLSOpts: tlsOpts, + }) + + // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // More info: + // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/server + // - https://book.kubebuilder.io/reference/metrics.html + metricsServerOptions := metricsserver.Options{ + BindAddress: metricsAddr, + SecureServing: secureMetrics, + TLSOpts: tlsOpts, + } + + if secureMetrics { + // FilterProvider is used to protect the metrics endpoint with authn/authz. + // These configurations ensure that only authorized users and service accounts + // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/filters#WithAuthenticationAndAuthorization + metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization + + // TODO(user): If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + } + + cfg := ctrl.GetConfigOrDie() + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme, + Metrics: metricsServerOptions, + WebhookServer: webhookServer, + HealthProbeBindAddress: probeAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "odh-model-controller.opendatahub.io", + // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily + // when the Manager ends. This requires the binary to immediately end when the + // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly + // speeds up voluntary leader transitions as the new leader don't have to wait + // LeaseDuration time first. + // + // In the default scaffold provided, the program ends immediately after + // the manager stops, so would be fine to enable this option. However, + // if you are doing or is intended to do any operation such as perform cleanups + // after the manager stops then its usage might be unsafe. + // LeaderElectionReleaseOnCancel: true, + Client: client.Options{ + Cache: &client.CacheOptions{ + DisableFor: []client.Object{&istiov1beta1.AuthorizationPolicy{}}, + }, + }, + Cache: cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + &corev1.Secret{}: { + Label: labels.SelectorFromSet(labels.Set{ + "opendatahub.io/managed": "true", + }), + }, + }, + }, + }) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + + kubeClient, kubeClientErr := kubernetes.NewForConfig(cfg) + if kubeClientErr != nil { + setupLog.Error(err, "unable to create clientset") + os.Exit(1) + } + + kserveWithMeshEnabled, kserveWithMeshEnabledErr := utils.VerifyIfComponentIsEnabled( + context.Background(), mgr.GetClient(), utils.KServeWithServiceMeshComponent) + if kserveWithMeshEnabledErr != nil { + setupLog.Error(kserveWithMeshEnabledErr, "could not determine if kserve have service mesh enabled") + } + + if err = (servingcontroller.NewInferenceServiceReconciler( + setupLog, + mgr.GetClient(), + mgr.GetScheme(), + mgr.GetAPIReader(), + kubeClient, + getEnvAsBool("MESH_DISABLED", false), + enableMRInferenceServiceReconcile, + getEnvAsBool("MR_SKIP_TLS_VERIFY", false), + cfg.BearerToken, + )).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "InferenceService") + os.Exit(1) + } + if err = (&corecontroller.SecretReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Secret") + os.Exit(1) + } + if err = (&corecontroller.ConfigMapReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "ConfigMap") + os.Exit(1) + } + if monitoringNS != "" { + setupLog.Info("Monitoring namespace provided, setting up monitoring controller.") + if err = (&servingcontroller.ServingRuntimeReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + MonitoringNS: monitoringNS, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "ServingRuntime") + os.Exit(1) + } + } + + nimState := os.Getenv("NIM_STATE") + signalHandlerCtx := ctrl.SetupSignalHandler() + if !slices.Contains([]string{"removed", ""}, nimState) { + if err = (&nim.AccountReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + KClient: kubeClient, + }).SetupWithManager(mgr, signalHandlerCtx); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "NIMAccount") + os.Exit(1) + } + } + + // nolint:goconst + if os.Getenv("ENABLE_WEBHOOKS") != "false" { + if err = webhooknimv1.SetupAccountWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "NIMAccount") + os.Exit(1) + } + } + // nolint:goconst + if os.Getenv("ENABLE_WEBHOOKS") != "false" { + if kserveWithMeshEnabled { + if err = webhookservingv1.SetupServiceWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "Knative Service") + os.Exit(1) + } + } else { + setupLog.Info("Skipping setup of Knative Service validating/mutating Webhook, " + + "because KServe Serverless setup seems to be disabled.") + } + } + // nolint:goconst + if os.Getenv("ENABLE_WEBHOOKS") != "false" { + if err = webhookservingv1beta1.SetupInferenceServiceWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "InferenceService") + os.Exit(1) + } + } + // +kubebuilder:scaffold:builder + + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up ready check") + os.Exit(1) + } + + setupLog.Info("starting manager") + if err := mgr.Start(signalHandlerCtx); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +} diff --git a/config/base/kustomization.yaml b/config/base/kustomization.yaml index 5ef7a42e..7bf66c58 100644 --- a/config/base/kustomization.yaml +++ b/config/base/kustomization.yaml @@ -1,8 +1,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - ../prometheus - - ../overlays/odh + - ../default namespace: opendatahub configMapGenerator: @@ -135,4 +134,7 @@ replacements: kind: ValidatingWebhookConfiguration name: validating-webhook-configuration fieldPaths: - - webhooks.0.clientConfig.service.namespace \ No newline at end of file + - webhooks.0.clientConfig.service.namespace + +patches: +- path: remove-namespace.yaml diff --git a/config/overlays/odh/odh_model_controller_manager_patch.yaml b/config/base/odh_model_controller_manager_patch.yaml similarity index 100% rename from config/overlays/odh/odh_model_controller_manager_patch.yaml rename to config/base/odh_model_controller_manager_patch.yaml diff --git a/config/base/remove-namespace.yaml b/config/base/remove-namespace.yaml new file mode 100644 index 00000000..b68652c5 --- /dev/null +++ b/config/base/remove-namespace.yaml @@ -0,0 +1,6 @@ +# Remove namespace resource as namespace will already exist. +$patch: delete +apiVersion: v1 +kind: Namespace +metadata: + name: system diff --git a/config/certmanager/certificate.yaml b/config/certmanager/certificate.yaml new file mode 100644 index 00000000..d13f9e33 --- /dev/null +++ b/config/certmanager/certificate.yaml @@ -0,0 +1,35 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: odh-model-controller + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: certificate + app.kubernetes.io/instance: serving-cert + app.kubernetes.io/component: certificate + app.kubernetes.io/created-by: odh-model-controller + app.kubernetes.io/part-of: odh-model-controller + app.kubernetes.io/managed-by: kustomize + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be replaced by kustomize + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/config/certmanager/kustomization.yaml b/config/certmanager/kustomization.yaml new file mode 100644 index 00000000..bebea5a5 --- /dev/null +++ b/config/certmanager/kustomization.yaml @@ -0,0 +1,5 @@ +resources: +- certificate.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/config/certmanager/kustomizeconfig.yaml b/config/certmanager/kustomizeconfig.yaml new file mode 100644 index 00000000..cf6f89e8 --- /dev/null +++ b/config/certmanager/kustomizeconfig.yaml @@ -0,0 +1,8 @@ +# This configuration is for teaching kustomize how to update name ref substitution +nameReference: +- kind: Issuer + group: cert-manager.io + fieldSpecs: + - kind: Certificate + group: cert-manager.io + path: spec/issuerRef/name diff --git a/config/crd/bases/nim.opendatahub.io_accounts.yaml b/config/crd/bases/nim.opendatahub.io_accounts.yaml index b86ad106..581149d7 100644 --- a/config/crd/bases/nim.opendatahub.io_accounts.yaml +++ b/config/crd/bases/nim.opendatahub.io_accounts.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.14.0 + controller-gen.kubebuilder.io/version: v0.16.4 name: accounts.nim.opendatahub.io spec: group: nim.opendatahub.io @@ -67,7 +67,6 @@ spec: the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. - TODO: this design is not final and this field is subject to change in the future. type: string kind: description: |- @@ -104,16 +103,8 @@ spec: properties: conditions: items: - description: "Condition contains details for one aspect of the current - state of this API Resource.\n---\nThis struct is intended for - direct use as an array at the field path .status.conditions. For - example,\n\n\n\ttype FooStatus struct{\n\t // Represents the - observations of a foo's current state.\n\t // Known .status.conditions.type - are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // - +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t - \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" - patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t - \ // other fields\n\t}" + description: Condition contains details for one aspect of the current + state of this API Resource. properties: lastTransitionTime: description: |- @@ -154,12 +145,7 @@ spec: - Unknown type: string type: - description: |- - type of condition in CamelCase or in foo.example.com/CamelCase. - --- - Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be - useful (see .node.status.conditions), the ability to deconflict is important. - The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + description: type of condition in CamelCase or in foo.example.com/CamelCase. maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string @@ -186,7 +172,6 @@ spec: the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. - TODO: this design is not final and this field is subject to change in the future. type: string kind: description: |- @@ -230,7 +215,6 @@ spec: the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. - TODO: this design is not final and this field is subject to change in the future. type: string kind: description: |- @@ -274,7 +258,6 @@ spec: the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. - TODO: this design is not final and this field is subject to change in the future. type: string kind: description: |- diff --git a/config/crd/external/route.openshift.io_routes.yaml b/config/crd/external/route.openshift.io_routes.yaml index 314ff2fc..ed0665f5 100644 --- a/config/crd/external/route.openshift.io_routes.yaml +++ b/config/crd/external/route.openshift.io_routes.yaml @@ -1,10 +1,12 @@ ---- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.8.0 - creationTimestamp: null + api-approved.openshift.io: https://github.com/openshift/api/pull/1228 + api.openshift.io/merged-by-featuregates: "true" + include.release.openshift.io/ibm-cloud-managed: "true" + include.release.openshift.io/self-managed-high-availability: "true" + release.openshift.io/feature-set: Default name: routes.route.openshift.io spec: group: route.openshift.io @@ -15,88 +17,420 @@ spec: singular: route scope: Namespaced versions: - - name: v1 + - additionalPrinterColumns: + - jsonPath: .status.ingress[0].host + name: Host + type: string + - jsonPath: .status.ingress[0].conditions[?(@.type=="Admitted")].status + name: Admitted + type: string + - jsonPath: .spec.to.name + name: Service + type: string + - jsonPath: .spec.tls.type + name: TLS + type: string + name: v1 schema: openAPIV3Schema: - description: "A route allows developers to expose services through an HTTP(S) - aware load balancing and proxy layer via a public DNS entry. The route may - further specify TLS options and a certificate, or specify a public CNAME - that the router should also accept for HTTP and HTTPS traffic. An administrator - typically configures their router to be visible outside the cluster firewall, - and may also add additional security, caching, or traffic controls on the - service content. Routers usually talk directly to the service endpoints. - \n Once a route is created, the `host` field may not be changed. Generally, - routers use the oldest route with a given host when resolving conflicts. - \n Routers are subject to additional customization and may support additional - controls via the annotations field. \n Because administrators may configure - multiple routers, the route status field is used to return information to - clients about the names and states of the route under each router. If a - client chooses a duplicate name, for instance, the route status conditions - are used to indicate the route cannot be chosen." + description: |- + A route allows developers to expose services through an HTTP(S) aware load balancing and proxy + layer via a public DNS entry. The route may further specify TLS options and a certificate, or + specify a public CNAME that the router should also accept for HTTP and HTTPS traffic. An + administrator typically configures their router to be visible outside the cluster firewall, and + may also add additional security, caching, or traffic controls on the service content. Routers + usually talk directly to the service endpoints. + + Once a route is created, the `host` field may not be changed. Generally, routers use the oldest + route with a given host when resolving conflicts. + + Routers are subject to additional customization and may support additional controls via the + annotations field. + + Because administrators may configure multiple routers, the route status field is used to + return information to clients about the names and states of the route under each router. + If a client chooses a duplicate name, for instance, the route status conditions are used + to indicate the route cannot be chosen. + + To enable HTTP/2 ALPN on a route it requires a custom + (non-wildcard) certificate. This prevents connection coalescing by + clients, notably web browsers. We do not support HTTP/2 ALPN on + routes that use the default certificate because of the risk of + connection re-use/coalescing. Routes that do not have their own + custom certificate will not be HTTP/2 ALPN-enabled on either the + frontend or the backend. + + Compatibility level 1: Stable within a major release for a minimum of 12 months or 3 minor releases (whichever is longer). properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: + allOf: + - anyOf: + - properties: + path: + maxLength: 0 + - properties: + tls: + enum: + - null + - not: + properties: + tls: + properties: + termination: + enum: + - passthrough + - anyOf: + - not: + properties: + host: + maxLength: 0 + - not: + properties: + wildcardPolicy: + enum: + - Subdomain description: spec is the desired state of the route properties: alternateBackends: - description: alternateBackends allows up to 3 additional backends - to be assigned to the route. Only the Service kind is allowed, and - it will be defaulted to Service. Use the weight field in RouteTargetReference - object to specify relative preference. + description: |- + alternateBackends allows up to 3 additional backends to be assigned to the route. + Only the Service kind is allowed, and it will be defaulted to Service. + Use the weight field in RouteTargetReference object to specify relative preference. items: - description: RouteTargetReference specifies the target that resolve - into endpoints. Only the 'Service' kind is allowed. Use 'weight' - field to emphasize one over others. + description: |- + RouteTargetReference specifies the target that resolve into endpoints. Only the 'Service' + kind is allowed. Use 'weight' field to emphasize one over others. properties: kind: + default: Service description: The kind of target that the route is referring to. Currently, only 'Service' is allowed + enum: + - Service + - "" type: string name: description: name of the service/target that is being referred to. e.g. name of the service + minLength: 1 type: string weight: - description: weight as an integer between 0 and 256, default - 1, that specifies the target's relative weight against other - target reference objects. 0 suppresses requests to this backend. + default: 100 + description: |- + weight as an integer between 0 and 256, default 100, that specifies the target's relative weight + against other target reference objects. 0 suppresses requests to this backend. format: int32 + maximum: 256 + minimum: 0 type: integer required: - kind - name - - weight type: object + maxItems: 3 type: array + x-kubernetes-list-map-keys: + - name + - kind + x-kubernetes-list-type: map host: - description: host is an alias/DNS that points to the service. Optional. - If not specified a route name will typically be automatically chosen. + description: |- + host is an alias/DNS that points to the service. Optional. + If not specified a route name will typically be automatically + chosen. Must follow DNS952 subdomain conventions. + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ type: string + httpHeaders: + description: httpHeaders defines policy for HTTP headers. + properties: + actions: + description: |- + actions specifies options for modifying headers and their values. + Note that this option only applies to cleartext HTTP connections + and to secure HTTP connections for which the ingress controller + terminates encryption (that is, edge-terminated or reencrypt + connections). Headers cannot be modified for TLS passthrough + connections. + Setting the HSTS (`Strict-Transport-Security`) header is not supported via actions. + `Strict-Transport-Security` may only be configured using the "haproxy.router.openshift.io/hsts_header" + route annotation, and only in accordance with the policy specified in Ingress.Spec.RequiredHSTSPolicies. + In case of HTTP request headers, the actions specified in spec.httpHeaders.actions on the Route will be executed after + the actions specified in the IngressController's spec.httpHeaders.actions field. + In case of HTTP response headers, the actions specified in spec.httpHeaders.actions on the IngressController will be + executed after the actions specified in the Route's spec.httpHeaders.actions field. + The headers set via this API will not appear in access logs. + Any actions defined here are applied after any actions related to the following other fields: + cache-control, spec.clientTLS, + spec.httpHeaders.forwardedHeaderPolicy, spec.httpHeaders.uniqueId, + and spec.httpHeaders.headerNameCaseAdjustments. + The following header names are reserved and may not be modified via this API: + Strict-Transport-Security, Proxy, Cookie, Set-Cookie. + Note that the total size of all net added headers *after* interpolating dynamic values + must not exceed the value of spec.tuningOptions.headerBufferMaxRewriteBytes on the + IngressController. Please refer to the documentation + for that API field for more details. + properties: + request: + description: |- + request is a list of HTTP request headers to modify. + Currently, actions may define to either `Set` or `Delete` headers values. + Actions defined here will modify the request headers of all requests made through a route. + These actions are applied to a specific Route defined within a cluster i.e. connections made through a route. + Currently, actions may define to either `Set` or `Delete` headers values. + Route actions will be executed after IngressController actions for request headers. + Actions are applied in sequence as defined in this list. + A maximum of 20 request header actions may be configured. + You can use this field to specify HTTP request headers that should be set or deleted + when forwarding connections from the client to your application. + Sample fetchers allowed are "req.hdr" and "ssl_c_der". + Converters allowed are "lower" and "base64". + Example header values: "%[req.hdr(X-target),lower]", "%{+Q}[ssl_c_der,base64]". + Any request header configuration applied directly via a Route resource using this API + will override header configuration for a header of the same name applied via + spec.httpHeaders.actions on the IngressController or route annotation. + Note: This field cannot be used if your route uses TLS passthrough. + items: + description: RouteHTTPHeader specifies configuration for + setting or deleting an HTTP header. + properties: + action: + description: action specifies actions to perform on + headers, such as setting or deleting headers. + properties: + set: + description: |- + set defines the HTTP header that should be set: added if it doesn't exist or replaced if it does. + This field is required when type is Set and forbidden otherwise. + properties: + value: + description: |- + value specifies a header value. + Dynamic values can be added. The value will be interpreted as an HAProxy format string as defined in + http://cbonte.github.io/haproxy-dconv/2.6/configuration.html#8.2.6 and may use HAProxy's %[] syntax and + otherwise must be a valid HTTP header value as defined in https://datatracker.ietf.org/doc/html/rfc7230#section-3.2. + The value of this field must be no more than 16384 characters in length. + Note that the total size of all net added headers *after* interpolating dynamic values + must not exceed the value of spec.tuningOptions.headerBufferMaxRewriteBytes on the + IngressController. + maxLength: 16384 + minLength: 1 + type: string + required: + - value + type: object + type: + description: |- + type defines the type of the action to be applied on the header. + Possible values are Set or Delete. + Set allows you to set HTTP request and response headers. + Delete allows you to delete HTTP request and response headers. + enum: + - Set + - Delete + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: set is required when type is Set, and forbidden + otherwise + rule: 'has(self.type) && self.type == ''Set'' ? has(self.set) + : !has(self.set)' + name: + description: |- + name specifies the name of a header on which to perform an action. Its value must be a valid HTTP header + name as defined in RFC 2616 section 4.2. + The name must consist only of alphanumeric and the following special characters, "-!#$%&'*+.^_`". + The following header names are reserved and may not be modified via this API: + Strict-Transport-Security, Proxy, Cookie, Set-Cookie. + It must be no more than 255 characters in length. + Header name must be unique. + maxLength: 255 + minLength: 1 + pattern: ^[-!#$%&'*+.0-9A-Z^_`a-z|~]+$ + type: string + x-kubernetes-validations: + - message: strict-transport-security header may not + be modified via header actions + rule: self.lowerAscii() != 'strict-transport-security' + - message: proxy header may not be modified via header + actions + rule: self.lowerAscii() != 'proxy' + - message: cookie header may not be modified via header + actions + rule: self.lowerAscii() != 'cookie' + - message: set-cookie header may not be modified via + header actions + rule: self.lowerAscii() != 'set-cookie' + required: + - action + - name + type: object + maxItems: 20 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + x-kubernetes-validations: + - message: Either the header value provided is not in correct + format or the sample fetcher/converter specified is not + allowed. The dynamic header value will be interpreted + as an HAProxy format string as defined in http://cbonte.github.io/haproxy-dconv/2.6/configuration.html#8.2.6 + and may use HAProxy's %[] syntax and otherwise must be + a valid HTTP header value as defined in https://datatracker.ietf.org/doc/html/rfc7230#section-3.2. + Sample fetchers allowed are req.hdr, ssl_c_der. Converters + allowed are lower, base64. + rule: self.all(key, key.action.type == "Delete" || (has(key.action.set) + && key.action.set.value.matches('^(?:%(?:%|(?:\\{[-+]?[QXE](?:,[-+]?[QXE])*\\})?\\[(?:req\\.hdr\\([0-9A-Za-z-]+\\)|ssl_c_der)(?:,(?:lower|base64))*\\])|[^%[:cntrl:]])+$'))) + response: + description: |- + response is a list of HTTP response headers to modify. + Currently, actions may define to either `Set` or `Delete` headers values. + Actions defined here will modify the response headers of all requests made through a route. + These actions are applied to a specific Route defined within a cluster i.e. connections made through a route. + Route actions will be executed before IngressController actions for response headers. + Actions are applied in sequence as defined in this list. + A maximum of 20 response header actions may be configured. + You can use this field to specify HTTP response headers that should be set or deleted + when forwarding responses from your application to the client. + Sample fetchers allowed are "res.hdr" and "ssl_c_der". + Converters allowed are "lower" and "base64". + Example header values: "%[res.hdr(X-target),lower]", "%{+Q}[ssl_c_der,base64]". + Note: This field cannot be used if your route uses TLS passthrough. + items: + description: RouteHTTPHeader specifies configuration for + setting or deleting an HTTP header. + properties: + action: + description: action specifies actions to perform on + headers, such as setting or deleting headers. + properties: + set: + description: |- + set defines the HTTP header that should be set: added if it doesn't exist or replaced if it does. + This field is required when type is Set and forbidden otherwise. + properties: + value: + description: |- + value specifies a header value. + Dynamic values can be added. The value will be interpreted as an HAProxy format string as defined in + http://cbonte.github.io/haproxy-dconv/2.6/configuration.html#8.2.6 and may use HAProxy's %[] syntax and + otherwise must be a valid HTTP header value as defined in https://datatracker.ietf.org/doc/html/rfc7230#section-3.2. + The value of this field must be no more than 16384 characters in length. + Note that the total size of all net added headers *after* interpolating dynamic values + must not exceed the value of spec.tuningOptions.headerBufferMaxRewriteBytes on the + IngressController. + maxLength: 16384 + minLength: 1 + type: string + required: + - value + type: object + type: + description: |- + type defines the type of the action to be applied on the header. + Possible values are Set or Delete. + Set allows you to set HTTP request and response headers. + Delete allows you to delete HTTP request and response headers. + enum: + - Set + - Delete + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: set is required when type is Set, and forbidden + otherwise + rule: 'has(self.type) && self.type == ''Set'' ? has(self.set) + : !has(self.set)' + name: + description: |- + name specifies the name of a header on which to perform an action. Its value must be a valid HTTP header + name as defined in RFC 2616 section 4.2. + The name must consist only of alphanumeric and the following special characters, "-!#$%&'*+.^_`". + The following header names are reserved and may not be modified via this API: + Strict-Transport-Security, Proxy, Cookie, Set-Cookie. + It must be no more than 255 characters in length. + Header name must be unique. + maxLength: 255 + minLength: 1 + pattern: ^[-!#$%&'*+.0-9A-Z^_`a-z|~]+$ + type: string + x-kubernetes-validations: + - message: strict-transport-security header may not + be modified via header actions + rule: self.lowerAscii() != 'strict-transport-security' + - message: proxy header may not be modified via header + actions + rule: self.lowerAscii() != 'proxy' + - message: cookie header may not be modified via header + actions + rule: self.lowerAscii() != 'cookie' + - message: set-cookie header may not be modified via + header actions + rule: self.lowerAscii() != 'set-cookie' + required: + - action + - name + type: object + maxItems: 20 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + x-kubernetes-validations: + - message: Either the header value provided is not in correct + format or the sample fetcher/converter specified is not + allowed. The dynamic header value will be interpreted + as an HAProxy format string as defined in http://cbonte.github.io/haproxy-dconv/2.6/configuration.html#8.2.6 + and may use HAProxy's %[] syntax and otherwise must be + a valid HTTP header value as defined in https://datatracker.ietf.org/doc/html/rfc7230#section-3.2. + Sample fetchers allowed are res.hdr, ssl_c_der. Converters + allowed are lower, base64. + rule: self.all(key, key.action.type == "Delete" || (has(key.action.set) + && key.action.set.value.matches('^(?:%(?:%|(?:\\{[-+]?[QXE](?:,[-+]?[QXE])*\\})?\\[(?:res\\.hdr\\([0-9A-Za-z-]+\\)|ssl_c_der)(?:,(?:lower|base64))*\\])|[^%[:cntrl:]])+$'))) + type: object + type: object path: - description: Path that the router watches for, to route traffic for + description: path that the router watches for, to route traffic for to the service. Optional + pattern: ^/ type: string port: - description: If specified, the port to be used by the router. Most - routers will use all endpoints exposed by the service by default - - set this value to instruct routers which port to use. + description: |- + If specified, the port to be used by the router. Most routers will use all + endpoints exposed by the service by default - set this value to instruct routers + which port to use. properties: targetPort: - anyOf: - - type: integer - - type: string + allOf: + - not: + enum: + - 0 + - not: + enum: + - "" + anyOf: null description: The target port on pods selected by the service this route points to. If this is a string, it will be looked up as a named port in the target endpoints port list. Required @@ -104,7 +438,49 @@ spec: required: - targetPort type: object + subdomain: + description: |- + subdomain is a DNS subdomain that is requested within the ingress controller's + domain (as a subdomain). If host is set this field is ignored. An ingress + controller may choose to ignore this suggested name, in which case the controller + will report the assigned name in the status.ingress array or refuse to admit the + route. If this value is set and the server does not support this field host will + be populated automatically. Otherwise host is left empty. The field may have + multiple parts separated by a dot, but not all ingress controllers may honor + the request. This field may not be changed after creation except by a user with + the update routes/custom-host permission. + + Example: subdomain `frontend` automatically receives the router subdomain + `apps.mycluster.com` to have a full hostname `frontend.apps.mycluster.com`. + maxLength: 253 + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + type: string tls: + allOf: + - anyOf: + - properties: + caCertificate: + maxLength: 0 + certificate: + maxLength: 0 + destinationCACertificate: + maxLength: 0 + key: + maxLength: 0 + - not: + properties: + termination: + enum: + - passthrough + - anyOf: + - properties: + destinationCACertificate: + maxLength: 0 + - not: + properties: + termination: + enum: + - edge description: The tls field provides the ability to configure certificates and termination for the route. properties: @@ -113,76 +489,119 @@ spec: contents type: string certificate: - description: certificate provides certificate contents + description: |- + certificate provides certificate contents. This should be a single serving certificate, not a certificate + chain. Do not include a CA certificate. type: string destinationCACertificate: - description: destinationCACertificate provides the contents of - the ca certificate of the final destination. When using reencrypt - termination this file should be provided in order to have routers - use it for health checks on the secure connection. If this field - is not specified, the router may provide its own destination - CA and perform hostname validation using the short service name - (service.namespace.svc), which allows infrastructure generated - certificates to automatically verify. + description: |- + destinationCACertificate provides the contents of the ca certificate of the final destination. When using reencrypt + termination this file should be provided in order to have routers use it for health checks on the secure connection. + If this field is not specified, the router may provide its own destination CA and perform hostname validation using + the short service name (service.namespace.svc), which allows infrastructure generated certificates to automatically + verify. type: string insecureEdgeTerminationPolicy: - description: "insecureEdgeTerminationPolicy indicates the desired - behavior for insecure connections to a route. While each router - may make its own decisions on which ports to expose, this is - normally port 80. \n * Allow - traffic is sent to the server - on the insecure port (default) * Disable - no traffic is allowed - on the insecure port. * Redirect - clients are redirected to - the secure port." + description: |- + insecureEdgeTerminationPolicy indicates the desired behavior for insecure connections to a route. While + each router may make its own decisions on which ports to expose, this is normally port 80. + + If a route does not specify insecureEdgeTerminationPolicy, then the default behavior is "None". + + * Allow - traffic is sent to the server on the insecure port (edge/reencrypt terminations only). + + * None - no traffic is allowed on the insecure port (default). + + * Redirect - clients are redirected to the secure port. + enum: + - Allow + - None + - Redirect + - "" type: string key: description: key provides key file contents type: string termination: - description: termination indicates termination type. + description: |- + termination indicates termination type. + + * edge - TLS termination is done by the router and http is used to communicate with the backend (default) + * passthrough - Traffic is sent straight to the destination without the router providing TLS termination + * reencrypt - TLS termination is done by the router and https is used to communicate with the backend + + Note: passthrough termination is incompatible with httpHeader actions + enum: + - edge + - reencrypt + - passthrough type: string required: - termination type: object + x-kubernetes-validations: + - message: 'cannot have both spec.tls.termination: passthrough and + spec.tls.insecureEdgeTerminationPolicy: Allow' + rule: 'has(self.termination) && has(self.insecureEdgeTerminationPolicy) + ? !((self.termination==''passthrough'') && (self.insecureEdgeTerminationPolicy==''Allow'')) + : true' to: - description: to is an object the route should use as the primary backend. - Only the Service kind is allowed, and it will be defaulted to Service. - If the weight field (0-256 default 1) is set to zero, no traffic - will be sent to this backend. + description: |- + to is an object the route should use as the primary backend. Only the Service kind + is allowed, and it will be defaulted to Service. If the weight field (0-256 default 100) + is set to zero, no traffic will be sent to this backend. properties: kind: + default: Service description: The kind of target that the route is referring to. Currently, only 'Service' is allowed + enum: + - Service + - "" type: string name: description: name of the service/target that is being referred to. e.g. name of the service + minLength: 1 type: string weight: - description: weight as an integer between 0 and 256, default 1, - that specifies the target's relative weight against other target - reference objects. 0 suppresses requests to this backend. + default: 100 + description: |- + weight as an integer between 0 and 256, default 100, that specifies the target's relative weight + against other target reference objects. 0 suppresses requests to this backend. format: int32 + maximum: 256 + minimum: 0 type: integer required: - kind - name - - weight type: object wildcardPolicy: - description: Wildcard policy if any for the route. Currently only - 'Subdomain' or 'None' is allowed. + default: None + description: |- + Wildcard policy if any for the route. + Currently only 'Subdomain' or 'None' is allowed. + enum: + - None + - Subdomain + - "" type: string required: - - host - to type: object + x-kubernetes-validations: + - message: header actions are not permitted when tls termination is passthrough. + rule: '!has(self.tls) || self.tls.termination != ''passthrough'' || + !has(self.httpHeaders)' status: description: status is the current state of the route properties: ingress: - description: ingress describes the places where the route may be exposed. - The list of ingress points may contain duplicate Host or RouterName - values. Routes are considered live once they are `Ready` + description: |- + ingress describes the places where the route may be exposed. The list of + ingress points may contain duplicate Host or RouterName values. Routes + are considered live once they are `Ready` items: description: RouteIngress holds information about the places where a route is exposed. @@ -190,8 +609,9 @@ spec: conditions: description: Conditions is the state of the route, may be empty. items: - description: RouteIngressCondition contains details for the - current condition of this route on a particular router. + description: |- + RouteIngressCondition contains details for the current condition of this route on a particular + router. properties: lastTransitionTime: description: RFC 3339 date and time when this condition @@ -203,31 +623,36 @@ spec: about last transition. type: string reason: - description: (brief) reason for the condition's last transition, - and is usually a machine and human readable constant + description: |- + (brief) reason for the condition's last transition, and is usually a machine and human + readable constant type: string status: - description: Status is the status of the condition. Can - be True, False, Unknown. + description: |- + Status is the status of the condition. + Can be True, False, Unknown. type: string type: - description: Type is the type of the condition. Currently - only Ready. + description: |- + Type is the type of the condition. + Currently only Admitted or UnservableInFutureVersions. type: string required: - status - type type: object type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map host: description: Host is the host string under which the route is exposed; this value is required type: string routerCanonicalHostname: - description: CanonicalHostname is the external host name for - the router that can be used as a CNAME for the host requested - for this route. This value is optional and may not be set - in all cases. + description: |- + CanonicalHostname is the external host name for the router that can be used as a CNAME + for the host requested for this route. This value is optional and may not be set in all cases. type: string routerName: description: Name is a name chosen by the router to identify @@ -239,18 +664,12 @@ spec: type: string type: object type: array - required: - - ingress + x-kubernetes-list-type: atomic type: object required: - spec - - status type: object served: true storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index d6cd1242..d175fa93 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -1,2 +1,20 @@ +# This kustomization.yaml is not intended to be run by itself, +# since it depends on service name and namespace that are out of this kustomize package. +# It should be run by config/default resources: - - bases/nim.opendatahub.io_accounts.yaml \ No newline at end of file +- bases/nim.opendatahub.io_accounts.yaml +# +kubebuilder:scaffold:crdkustomizeresource + +patches: +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. +# patches here are for enabling the conversion webhook for each CRD +# +kubebuilder:scaffold:crdkustomizewebhookpatch + +# [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. +# patches here are for enabling the CA injection for each CRD +# +kubebuilder:scaffold:crdkustomizecainjectionpatch + +# [WEBHOOK] To enable webhook, uncomment the following section +# the following config is for teaching kustomize how to do kustomization for CRDs. +configurations: +- kustomizeconfig.yaml diff --git a/config/crd/kustomizeconfig.yaml b/config/crd/kustomizeconfig.yaml new file mode 100644 index 00000000..fd57a0e6 --- /dev/null +++ b/config/crd/kustomizeconfig.yaml @@ -0,0 +1,19 @@ +# This file is for teaching kustomize how to replace name and namespace references in CRD +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/name + +namespace: +- kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/namespace + create: false + +varReference: +- path: metadata/annotations diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 8f551561..17f63d37 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -1,7 +1,178 @@ +# Adds namespace to all resources. +namespace: odh-model-controller-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +# namePrefix: odh-model-controller- + +# Labels to add to all resources and selectors. +#labels: +#- includeSelectors: true +# pairs: +# someName: someValue + resources: - - ../crd - - ../rbac - - ../manager - - ../webhook - - ../runtimes - - networkpolicy.yaml \ No newline at end of file +- ../crd +- ../rbac +- ../manager +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +- ../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +#- ../certmanager +# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. +- ../prometheus +# [METRICS] Expose the controller manager metrics service. +- metrics_service.yaml +# [NETWORK POLICY] Protect the /metrics endpoint and Webhook Server with NetworkPolicy. +# Only Pod(s) running a namespace labeled with 'metrics: enabled' will be able to gather the metrics. +# Only CR(s) which requires webhooks and are applied on namespaces labeled with 'webhooks: enabled' will +# be able to communicate with the Webhook Server. +- ../network-policy +- ../runtimes + +# Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager +patches: +# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. +# More info: https://book.kubebuilder.io/reference/metrics +- path: manager_metrics_patch.yaml + target: + kind: Deployment + +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +- path: manager_webhook_patch.yaml + +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +# Uncomment the following replacements to add the cert-manager CA injection annotations +#replacements: +# - source: # Uncomment the following block if you have any webhook +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.name # Name of the service +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - source: +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.namespace # Namespace of the service +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# +# - source: # Uncomment the following block if you have a ValidatingWebhook (--programmatic-validation) +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # This name should match the one in certificate.yaml +# fieldPath: .metadata.namespace # Namespace of the certificate CR +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # This name should match the one in certificate.yaml +# fieldPath: .metadata.name +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true +# +# - source: # Uncomment the following block if you have a DefaultingWebhook (--defaulting ) +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # This name should match the one in certificate.yaml +# fieldPath: .metadata.namespace # Namespace of the certificate CR +# targets: +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # This name should match the one in certificate.yaml +# fieldPath: .metadata.name +# targets: +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true +# +# - source: # Uncomment the following block if you have a ConversionWebhook (--conversion) +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # This name should match the one in certificate.yaml +# fieldPath: .metadata.namespace # Namespace of the certificate CR +# targets: +# - select: +# kind: CustomResourceDefinition +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # This name should match the one in certificate.yaml +# fieldPath: .metadata.name +# targets: +# - select: +# kind: CustomResourceDefinition +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true diff --git a/config/default/manager_metrics_patch.yaml b/config/default/manager_metrics_patch.yaml new file mode 100644 index 00000000..dee2761d --- /dev/null +++ b/config/default/manager_metrics_patch.yaml @@ -0,0 +1,5 @@ +# This patch adds the args to allow exposing the metrics endpoint using HTTPS +# TODO: restore to 8443 port +- op: add + path: /spec/template/spec/containers/0/args/0 + value: --metrics-bind-address=:8080 diff --git a/config/default/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml new file mode 100644 index 00000000..3f9e0e84 --- /dev/null +++ b/config/default/manager_webhook_patch.yaml @@ -0,0 +1,26 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: odh-model-controller + namespace: system +# labels: +# app.kubernetes.io/name: odh-model-controller +# app.kubernetes.io/managed-by: kustomize +spec: + template: + spec: + containers: + - name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: odh-model-controller-webhook-cert diff --git a/config/default/metrics_service.yaml b/config/default/metrics_service.yaml new file mode 100644 index 00000000..feff3f86 --- /dev/null +++ b/config/default/metrics_service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: odh-model-controller +# app.kubernetes.io/name: odh-model-controller +# app.kubernetes.io/managed-by: kustomize + name: odh-model-controller-metrics-service + namespace: system +spec: + ports: + - name: http # TODO: Restore to http + port: 8080 # TODO: Use TLS and change to 8443 + protocol: TCP + targetPort: 8080 # TODO: Use TLS and change to 8443 + selector: + control-plane: odh-model-controller diff --git a/config/default/networkpolicy.yaml b/config/default/networkpolicy.yaml deleted file mode 100644 index 06ea22e0..00000000 --- a/config/default/networkpolicy.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: odh-model-controller -spec: - ingress: - - ports: - - port: 9443 - protocol: TCP - podSelector: - matchLabels: - app: odh-model-controller - control-plane: odh-model-controller diff --git a/config/manager/controller_manager_config.yaml b/config/manager/controller_manager_config.yaml deleted file mode 100644 index a0ae34b7..00000000 --- a/config/manager/controller_manager_config.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 -kind: ControllerManagerConfig -health: - healthProbeBindAddress: :8081 -metrics: - bindAddress: 127.0.0.1:8080 -webhook: - port: 9443 -leaderElection: - leaderElect: true - resourceName: odh-model-controller diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index f033e3ee..5c5f0b84 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,11 +1,2 @@ resources: - manager.yaml - -generatorOptions: - disableNameSuffixHash: true - -configMapGenerator: -- files: - - controller_manager_config.yaml - name: manager-config - diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index bb156abb..59b4907c 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -1,7 +1,17 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: odh-model-controller + app.kubernetes.io/managed-by: kustomize + name: system +--- apiVersion: apps/v1 kind: Deployment metadata: name: odh-model-controller + namespace: system labels: control-plane: odh-model-controller app: odh-model-controller @@ -18,98 +28,102 @@ spec: control-plane: odh-model-controller app: odh-model-controller spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: control-plane - operator: In - values: - - odh-model-controller - topologyKey: kubernetes.io/hostname + # TODO(user): Uncomment the following code to configure the nodeAffinity expression + # according to the platforms which are supported by your solution. + # It is considered best practice to support multiple architectures. You can + # build your manager image using the makefile target docker-buildx. + # affinity: + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/arch + # operator: In + # values: + # - amd64 + # - arm64 + # - ppc64le + # - s390x + # - key: kubernetes.io/os + # operator: In + # values: + # - linux securityContext: runAsNonRoot: true + # TODO(user): For common cases that do not require escalating privileges + # it is recommended to ensure that all your Pods/Containers are restrictive. + # More info: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted + # Please uncomment the following code if your project does NOT have to work on old Kubernetes + # versions < 1.19 or on vendors versions which do NOT support this field by default (i.e. Openshift < 4.11 ). + # seccompProfile: + # type: RuntimeDefault containers: - - command: - - /manager - args: - - --leader-elect - image: controller:latest - name: manager - imagePullPolicy: Always - securityContext: - allowPrivilegeEscalation: false - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: AUTH_AUDIENCE - valueFrom: - configMapKeyRef: - name: auth-refs - key: AUTH_AUDIENCE - optional: true - - name: AUTHORINO_LABEL - valueFrom: - configMapKeyRef: - name: auth-refs - key: AUTHORINO_LABEL - optional: true - - name: CONTROL_PLANE_NAME - valueFrom: - configMapKeyRef: - name: service-mesh-refs - key: CONTROL_PLANE_NAME - optional: true - - name: MESH_NAMESPACE - valueFrom: - configMapKeyRef: - name: service-mesh-refs - key: MESH_NAMESPACE - optional: true - - name: NIM_STATE - valueFrom: - configMapKeyRef: - name: odh-model-controller-parameters - key: nim-state - optional: true - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - # TODO(user): Configure the resources accordingly based on the project requirements. - # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - resources: - limits: - cpu: 500m - memory: 2Gi - requests: - cpu: 10m - memory: 64Mi - ports: - - containerPort: 9443 - name: webhook-server - protocol: TCP - volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: cert - readOnly: true + - command: + - /manager + args: + - --leader-elect + - --health-probe-bind-address=:8081 + image: controller:latest + name: manager + imagePullPolicy: Always + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: AUTH_AUDIENCE + valueFrom: + configMapKeyRef: + name: auth-refs + key: AUTH_AUDIENCE + optional: true + - name: AUTHORINO_LABEL + valueFrom: + configMapKeyRef: + name: auth-refs + key: AUTHORINO_LABEL + optional: true + - name: CONTROL_PLANE_NAME + valueFrom: + configMapKeyRef: + name: service-mesh-refs + key: CONTROL_PLANE_NAME + optional: true + - name: MESH_NAMESPACE + valueFrom: + configMapKeyRef: + name: service-mesh-refs + key: MESH_NAMESPACE + optional: true + - name: NIM_STATE + valueFrom: + configMapKeyRef: + name: odh-model-controller-parameters + key: nim-state + optional: true + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 500m + memory: 2Gi + requests: + cpu: 10m + memory: 64Mi serviceAccountName: odh-model-controller terminationGracePeriodSeconds: 10 - volumes: - - name: cert - secret: - defaultMode: 420 - secretName: odh-model-controller-webhook-cert diff --git a/config/manager/namespace.yaml b/config/manager/namespace.yaml deleted file mode 100644 index 395003f3..00000000 --- a/config/manager/namespace.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - labels: - control-plane: odh-model-controller - name: system diff --git a/config/network-policy/allow-metrics-traffic.yaml b/config/network-policy/allow-metrics-traffic.yaml new file mode 100644 index 00000000..301b2519 --- /dev/null +++ b/config/network-policy/allow-metrics-traffic.yaml @@ -0,0 +1,26 @@ +# This NetworkPolicy allows ingress traffic +# with Pods running on namespaces labeled with 'metrics: enabled'. Only Pods on those +# namespaces are able to gathering data from the metrics endpoint. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + app.kubernetes.io/name: odh-model-controller + app.kubernetes.io/managed-by: kustomize + name: allow-metrics-traffic + namespace: system +spec: + podSelector: + matchLabels: + control-plane: controller-manager + policyTypes: + - Ingress + ingress: + # This allows ingress traffic from any namespace with the label metrics: enabled + - from: + - namespaceSelector: + matchLabels: + metrics: enabled # Only from namespaces with this label + ports: + - port: 8443 + protocol: TCP diff --git a/config/network-policy/allow-webhook-traffic.yaml b/config/network-policy/allow-webhook-traffic.yaml new file mode 100644 index 00000000..61b5798c --- /dev/null +++ b/config/network-policy/allow-webhook-traffic.yaml @@ -0,0 +1,26 @@ +# This NetworkPolicy allows ingress traffic to your webhook server running +# as part of the controller-manager from specific namespaces and pods. CR(s) which uses webhooks +# will only work when applied in namespaces labeled with 'webhook: enabled' +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + app.kubernetes.io/name: odh-model-controller + app.kubernetes.io/managed-by: kustomize + name: odh-model-controller # Original scaffolded name is allow-webhook-traffic + namespace: system +spec: + podSelector: + matchLabels: + control-plane: odh-model-controller + policyTypes: + - Ingress + ingress: + # This allows ingress traffic from any namespace with the label webhook: enabled + - from: + - namespaceSelector: + matchLabels: + webhook: enabled # Only from namespaces with this label + ports: + - port: 443 + protocol: TCP diff --git a/config/network-policy/kustomization.yaml b/config/network-policy/kustomization.yaml new file mode 100644 index 00000000..638c3ef7 --- /dev/null +++ b/config/network-policy/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- allow-webhook-traffic.yaml +#- allow-metrics-traffic.yaml diff --git a/config/overlays/dev/kustomization.yaml b/config/overlays/dev/kustomization.yaml deleted file mode 100644 index 30e98395..00000000 --- a/config/overlays/dev/kustomization.yaml +++ /dev/null @@ -1,28 +0,0 @@ -resources: -- ../../crd/external -- ../../default - -patches: -- path: odh_model_controller_manager_patch.yaml - -namespace: default -configMapGenerator: -- envs: - - params.env - name: odh-model-controller-parameters -generatorOptions: - disableNameSuffixHash: true - - -replacements: -- source: - kind: ConfigMap - version: v1 - name: odh-model-controller-parameters - fieldPath: data.monitoring-namespace - targets: - - select: - kind: Deployment - name: odh-model-controller - fieldPaths: - - spec.template.spec.containers.0.env.0.value \ No newline at end of file diff --git a/config/overlays/dev/odh_model_controller_manager_patch.yaml b/config/overlays/dev/odh_model_controller_manager_patch.yaml deleted file mode 100644 index dd55dead..00000000 --- a/config/overlays/dev/odh_model_controller_manager_patch.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: odh-model-controller -spec: - replicas: 1 - template: - spec: - containers: - - args: - - --leader-elect - - --model-registry-inference-reconcile - - "--monitoring-namespace" - - "$(MONITORING_NS)" - env: - - name: MONITORING_NS - value: $(monitoring-namespace) - name: manager - imagePullPolicy: IfNotPresent diff --git a/config/overlays/dev/params.env b/config/overlays/dev/params.env deleted file mode 100644 index b9037d39..00000000 --- a/config/overlays/dev/params.env +++ /dev/null @@ -1,2 +0,0 @@ -monitoring-namespace=default -metadata.namespace=default diff --git a/config/overlays/odh/kustomization.yaml b/config/overlays/odh/kustomization.yaml deleted file mode 100644 index 880585aa..00000000 --- a/config/overlays/odh/kustomization.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - ../../default - -patches: -- path: odh_model_controller_manager_patch.yaml - -configurations: - - params.yaml diff --git a/config/overlays/odh/params.yaml b/config/overlays/odh/params.yaml deleted file mode 100644 index 3953383d..00000000 --- a/config/overlays/odh/params.yaml +++ /dev/null @@ -1,9 +0,0 @@ -varReference: - - path: metadata/name - kind: ClusterRoleBinding - - path: spec/template/spec/containers[]/image - kind: Deployment - - path: objects[]/spec/containers[]/image - kind: Template - - path: webhooks/clientConfig[]/service/namespace - kind: ValidatingWebhookConfiguration diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml index d556b996..ed137168 100644 --- a/config/prometheus/kustomization.yaml +++ b/config/prometheus/kustomization.yaml @@ -1,2 +1,2 @@ resources: - - monitor.yaml +- monitor.yaml diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml index a5cd6ad2..cc406d95 100644 --- a/config/prometheus/monitor.yaml +++ b/config/prometheus/monitor.yaml @@ -4,14 +4,26 @@ kind: ServiceMonitor metadata: labels: control-plane: odh-model-controller +# app.kubernetes.io/name: odh-model-controller +# app.kubernetes.io/managed-by: kustomize name: odh-model-controller-metrics-monitor + namespace: system spec: endpoints: - path: /metrics - port: https + port: https # Ensure this is the name of the port that exposes HTTPS metrics scheme: https bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token tlsConfig: + # Enabling option insecureSkipVerify is not recommended for production since it disables + # certificate verification. This poses a significant security risk by making the system vulnerable to + # man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between + # Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data, + # compromising the integrity and confidentiality of the information. + # Please use the following options for secure configurations: + # caFile: /etc/metrics-certs/ca.crt + # certFile: /etc/metrics-certs/tls.crt + # keyFile: /etc/metrics-certs/tls.key insecureSkipVerify: false selector: matchLabels: diff --git a/config/rbac/account_editor_role.yaml b/config/rbac/account_editor_role.yaml new file mode 100644 index 00000000..8316e9ef --- /dev/null +++ b/config/rbac/account_editor_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to edit accounts. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: odh-model-controller + app.kubernetes.io/managed-by: kustomize + name: account-editor-role +rules: +- apiGroups: + - nim.opendatahub.io + resources: + - accounts + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - nim.opendatahub.io + resources: + - accounts/status + verbs: + - get diff --git a/config/rbac/account_viewer_role.yaml b/config/rbac/account_viewer_role.yaml new file mode 100644 index 00000000..e3389dec --- /dev/null +++ b/config/rbac/account_viewer_role.yaml @@ -0,0 +1,23 @@ +# permissions for end users to view accounts. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: odh-model-controller + app.kubernetes.io/managed-by: kustomize + name: account-viewer-role +rules: +- apiGroups: + - nim.opendatahub.io + resources: + - accounts + verbs: + - get + - list + - watch +- apiGroups: + - nim.opendatahub.io + resources: + - accounts/status + verbs: + - get diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml deleted file mode 100644 index 237ac482..00000000 --- a/config/rbac/auth_proxy_service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - control-plane: odh-model-controller - name: odh-model-controller-metrics-service - namespace: system -spec: - ports: - - name: metrics - port: 8080 - protocol: TCP - targetPort: 8080 - selector: - control-plane: odh-model-controller diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 294db7e3..15e38921 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -1,20 +1,30 @@ resources: - # All RBAC will be applied under this service account in - # the deployment namespace. You may comment out this resource - # if your manager will use a service account that exists at - # runtime. Be sure to update RoleBinding and ClusterRoleBinding - # subjects if changing service account names. - - service_account.yaml - - role.yaml - - role_binding.yaml - - leader_election_role.yaml - - leader_election_role_binding.yaml - # Comment the following 4 lines if you want to disable - # the auth proxy (https://github.com/brancz/kube-rbac-proxy) - # which protects your /metrics endpoint. - - auth_proxy_service.yaml - - auth_proxy_role.yaml - - auth_proxy_role_binding.yaml - - auth_proxy_client_clusterrole.yaml - - kserve_prometheus_clusterrole.yaml - +- kserve_prometheus_clusterrole.yaml +# All RBAC will be applied under this service account in +# the deployment namespace. You may comment out this resource +# if your manager will use a service account that exists at +# runtime. Be sure to update RoleBinding and ClusterRoleBinding +# subjects if changing service account names. +- service_account.yaml +- role.yaml +- role_binding.yaml +- leader_election_role.yaml +- leader_election_role_binding.yaml +# The following RBAC configurations are used to protect +# the metrics endpoint with authn/authz. These configurations +# ensure that only authorized users and service accounts +# can access the metrics endpoint. Comment the following +# permissions if you want to disable this protection. +# More info: https://book.kubebuilder.io/reference/metrics.html +#- metrics_auth_role.yaml +#- metrics_auth_role_binding.yaml +- metrics_reader_role.yaml +# For each CRD, "Editor" and "Viewer" roles are scaffolded by +# default, aiding admins in cluster management. Those roles are +# not used by the Project itself. You can comment the following lines +# if you do not want those helpers be installed with your Project. +#- account_editor_role.yaml +#- account_viewer_role.yaml +# The following RBAC configurations are for Authorino +- auth_proxy_role.yaml +- auth_proxy_role_binding.yaml \ No newline at end of file diff --git a/config/rbac/leader_election_role.yaml b/config/rbac/leader_election_role.yaml index 9221419f..3985d7c6 100644 --- a/config/rbac/leader_election_role.yaml +++ b/config/rbac/leader_election_role.yaml @@ -2,36 +2,39 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: +# labels: +# app.kubernetes.io/name: odh-model-controller +# app.kubernetes.io/managed-by: kustomize name: leader-election-role rules: - - apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch diff --git a/config/rbac/leader_election_role_binding.yaml b/config/rbac/leader_election_role_binding.yaml index cb4a9c6f..94e457a7 100644 --- a/config/rbac/leader_election_role_binding.yaml +++ b/config/rbac/leader_election_role_binding.yaml @@ -1,11 +1,15 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: +# labels: +# app.kubernetes.io/name: odh-model-controller +# app.kubernetes.io/managed-by: kustomize name: leader-election-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: leader-election-role subjects: - - kind: ServiceAccount - name: odh-model-controller +- kind: ServiceAccount + name: odh-model-controller + namespace: system diff --git a/config/rbac/metrics_auth_role.yaml b/config/rbac/metrics_auth_role.yaml new file mode 100644 index 00000000..32d2e4ec --- /dev/null +++ b/config/rbac/metrics_auth_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/config/rbac/metrics_auth_role_binding.yaml b/config/rbac/metrics_auth_role_binding.yaml new file mode 100644 index 00000000..e775d67f --- /dev/null +++ b/config/rbac/metrics_auth_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metrics-auth-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/auth_proxy_client_clusterrole.yaml b/config/rbac/metrics_reader_role.yaml similarity index 61% rename from config/rbac/auth_proxy_client_clusterrole.yaml rename to config/rbac/metrics_reader_role.yaml index 07f43829..51a75db4 100644 --- a/config/rbac/auth_proxy_client_clusterrole.yaml +++ b/config/rbac/metrics_reader_role.yaml @@ -3,7 +3,7 @@ kind: ClusterRole metadata: name: metrics-reader rules: - - nonResourceURLs: - - "/metrics" - verbs: - - get +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6970313d..66c5a3cb 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -11,6 +11,7 @@ rules: - configmaps - secrets - serviceaccounts + - services verbs: - create - delete @@ -32,18 +33,6 @@ rules: - patch - update - watch -- apiGroups: - - "" - resources: - - services - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - authorino.kuadrant.io resources: @@ -74,6 +63,7 @@ rules: - watch - apiGroups: - extensions + - networking.k8s.io resources: - ingresses verbs: @@ -136,17 +126,6 @@ rules: - networking.istio.io resources: - virtualservices - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - networking.istio.io - resources: - virtualservices/finalizers verbs: - create @@ -156,14 +135,6 @@ rules: - patch - update - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch - apiGroups: - networking.k8s.io resources: @@ -180,6 +151,7 @@ rules: - nim.opendatahub.io resources: - accounts + - accounts/status verbs: - get - list @@ -191,15 +163,6 @@ rules: - accounts/finalizers verbs: - update -- apiGroups: - - nim.opendatahub.io - resources: - - accounts/status - verbs: - - get - - list - - update - - watch - apiGroups: - rbac.authorization.k8s.io resources: @@ -254,14 +217,6 @@ rules: - serving.kserve.io resources: - inferenceservices - verbs: - - get - - list - - update - - watch -- apiGroups: - - serving.kserve.io - resources: - inferenceservices/finalizers verbs: - get @@ -283,13 +238,7 @@ rules: resources: - servingruntimes/finalizers verbs: - - create - - delete - - get - - list - - patch - update - - watch - apiGroups: - telemetry.istio.io resources: diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml index 6df63e05..a77f0225 100644 --- a/config/rbac/role_binding.yaml +++ b/config/rbac/role_binding.yaml @@ -1,11 +1,15 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: +# labels: +# app.kubernetes.io/name: odh-model-controller +# app.kubernetes.io/managed-by: kustomize name: odh-model-controller-rolebinding-opendatahub roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: odh-model-controller-role subjects: - - kind: ServiceAccount - name: odh-model-controller +- kind: ServiceAccount + name: odh-model-controller + namespace: system diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml index 2f4a4fb0..88cf88e0 100644 --- a/config/rbac/service_account.yaml +++ b/config/rbac/service_account.yaml @@ -1,4 +1,8 @@ apiVersion: v1 kind: ServiceAccount metadata: +# labels: +# app.kubernetes.io/name: odh-model-controller +# app.kubernetes.io/managed-by: kustomize name: odh-model-controller + namespace: system diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml new file mode 100644 index 00000000..6266cc85 --- /dev/null +++ b/config/samples/kustomization.yaml @@ -0,0 +1,4 @@ +## Append samples of your project ## +resources: +- nim_v1_account.yaml +# +kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/samples/nim_v1_account.yaml b/config/samples/nim_v1_account.yaml new file mode 100644 index 00000000..b0693491 --- /dev/null +++ b/config/samples/nim_v1_account.yaml @@ -0,0 +1,9 @@ +apiVersion: nim.opendatahub.io/v1 +kind: Account +metadata: + labels: + app.kubernetes.io/name: odh-model-controller + app.kubernetes.io/managed-by: kustomize + name: account-sample +spec: + # TODO(user): Add fields here diff --git a/config/webhook/field_patch.yaml b/config/webhook/field_patch.yaml deleted file mode 100644 index f1b1dc24..00000000 --- a/config/webhook/field_patch.yaml +++ /dev/null @@ -1,3 +0,0 @@ -- op: replace - path: /metadata/name - value: validating.odh-model-controller.opendatahub.io \ No newline at end of file diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml index 5fec9aa6..5f5d3f74 100644 --- a/config/webhook/kustomization.yaml +++ b/config/webhook/kustomization.yaml @@ -1,23 +1,46 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - resources: - manifests.yaml - service.yaml patches: - - path: webhook_patch.yaml - target: - group: admissionregistration.k8s.io - kind: ValidatingWebhookConfiguration - name: validating-webhook-configuration - version: v1 - - path: field_patch.yaml - target: - group: admissionregistration.k8s.io - kind: ValidatingWebhookConfiguration - name: validating-webhook-configuration - version: v1 +- patch: |- + apiVersion: admissionregistration.k8s.io/v1 + kind: ValidatingWebhookConfiguration + metadata: + name: validating.odh-model-controller.opendatahub.io + annotations: + service.beta.openshift.io/inject-cabundle: true + webhooks: + - name: validating.ksvc.odh-model-controller.opendatahub.io + clientConfig: + service: + name: odh-model-controller-webhook-service + objectSelector: + matchExpressions: + - key: serving.kserve.io/inferenceservice + operator: Exists + - name: validating.nim.account.odh-model-controller.opendatahub.io + clientConfig: + service: + name: odh-model-controller-webhook-service + - name: validating.isvc.odh-model-controller.opendatahub.io + clientConfig: + service: + name: odh-model-controller-webhook-service + target: + group: admissionregistration.k8s.io + kind: ValidatingWebhookConfiguration + name: validating-webhook-configuration + version: v1 +- patch: |- + - op: replace + path: /metadata/name + value: validating.odh-model-controller.opendatahub.io + target: + group: admissionregistration.k8s.io + kind: ValidatingWebhookConfiguration + name: validating-webhook-configuration + version: v1 configurations: -- kustomizeconfig.yaml +- kustomizeconfig.yaml diff --git a/config/webhook/kustomizeconfig.yaml b/config/webhook/kustomizeconfig.yaml index de569105..206316e5 100644 --- a/config/webhook/kustomizeconfig.yaml +++ b/config/webhook/kustomizeconfig.yaml @@ -1,5 +1,16 @@ -# the following config is for teaching kustomize where to look at when substituting vars. +# the following config is for teaching kustomize where to look at when substituting nameReference. # It requires kustomize v2.1.0 or newer to work properly. +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + - kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + namespace: - kind: MutatingWebhookConfiguration group: admissionregistration.k8s.io @@ -9,6 +20,3 @@ namespace: group: admissionregistration.k8s.io path: webhooks/clientConfig/service/namespace create: true - -varReference: -- path: metadata/annotations diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 64d06815..6bf3b9f5 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -5,23 +5,24 @@ metadata: name: validating-webhook-configuration webhooks: - admissionReviewVersions: - - v1beta1 + - v1 clientConfig: service: name: webhook-service namespace: system - path: /validate-isvc-odh-service + path: /validate-nim-opendatahub-io-v1-account failurePolicy: Fail - name: validating.isvc.odh-model-controller.opendatahub.io + name: validating.nim.account.odh-model-controller.opendatahub.io rules: - apiGroups: - - serving.kserve.io + - nim.opendatahub.io apiVersions: - - v1beta1 + - v1 operations: - CREATE + - UPDATE resources: - - inferenceservices + - accounts sideEffects: None - admissionReviewVersions: - v1 @@ -48,17 +49,16 @@ webhooks: service: name: webhook-service namespace: system - path: /validate-nim-opendatahub-io-v1-account + path: /validate-serving-kserve-io-v1beta1-inferenceservice failurePolicy: Fail - name: validating.nim.account.odh-model-controller.opendatahub.io + name: validating.isvc.odh-model-controller.opendatahub.io rules: - apiGroups: - - nim.opendatahub.io + - serving.kserve.io apiVersions: - - v1 + - v1beta1 operations: - CREATE - - UPDATE resources: - - accounts + - inferenceservices sideEffects: None diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml index 2626721c..d93e1b4f 100644 --- a/config/webhook/service.yaml +++ b/config/webhook/service.yaml @@ -1,12 +1,17 @@ apiVersion: v1 kind: Service metadata: - name: odh-model-controller-webhook-service +# labels: +# app.kubernetes.io/name: odh-model-controller +# app.kubernetes.io/managed-by: kustomize annotations: service.beta.openshift.io/serving-cert-secret-name: odh-model-controller-webhook-cert + name: odh-model-controller-webhook-service + namespace: system spec: ports: - port: 443 - targetPort: webhook-server + protocol: TCP + targetPort: 9443 selector: control-plane: odh-model-controller diff --git a/config/webhook/webhook_patch.yaml b/config/webhook/webhook_patch.yaml deleted file mode 100644 index 51c3307d..00000000 --- a/config/webhook/webhook_patch.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: validating.odh-model-controller.opendatahub.io - annotations: - service.beta.openshift.io/inject-cabundle: true -webhooks: - - name: validating.ksvc.odh-model-controller.opendatahub.io - clientConfig: - service: - name: odh-model-controller-webhook-service - objectSelector: - matchExpressions: - - key: serving.kserve.io/inferenceservice - operator: Exists - - name: validating.nim.account.odh-model-controller.opendatahub.io - clientConfig: - service: - name: odh-model-controller-webhook-service - - name: validating.isvc.odh-model-controller.opendatahub.io - clientConfig: - service: - name: odh-model-controller-webhook-service diff --git a/controllers/inferenceservice_controller.go b/controllers/inferenceservice_controller.go deleted file mode 100644 index 8c90dadb..00000000 --- a/controllers/inferenceservice_controller.go +++ /dev/null @@ -1,229 +0,0 @@ -/* - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -import ( - "context" - - "github.com/go-logr/logr" - kservev1alpha1 "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" - kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - authorinov1beta2 "github.com/kuadrant/authorino/api/v1beta2" - "github.com/opendatahub-io/odh-model-controller/controllers/constants" - "github.com/opendatahub-io/odh-model-controller/controllers/reconcilers" - "github.com/opendatahub-io/odh-model-controller/controllers/utils" - routev1 "github.com/openshift/api/route/v1" - monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" - corev1 "k8s.io/api/core/v1" - networkingv1 "k8s.io/api/networking/v1" - authv1 "k8s.io/api/rbac/v1" - apierrs "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" - - // "sigs.k8s.io/controller-runtime/pkg/builder" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/reconcile" -) - -// OpenshiftInferenceServiceReconciler holds the controller configuration. -type OpenshiftInferenceServiceReconciler struct { - client client.Client - clientReader client.Reader - log logr.Logger - MeshDisabled bool - mmISVCReconciler *reconcilers.ModelMeshInferenceServiceReconciler - kserveServerlessISVCReconciler *reconcilers.KserveServerlessInferenceServiceReconciler - kserveRawISVCReconciler *reconcilers.KserveRawInferenceServiceReconciler -} - -func NewOpenshiftInferenceServiceReconciler(client client.Client, clientReader client.Reader, log logr.Logger, meshDisabled bool) *OpenshiftInferenceServiceReconciler { - return &OpenshiftInferenceServiceReconciler{ - client: client, - clientReader: clientReader, - log: log, - MeshDisabled: meshDisabled, - mmISVCReconciler: reconcilers.NewModelMeshInferenceServiceReconciler(client), - kserveServerlessISVCReconciler: reconcilers.NewKServeServerlessInferenceServiceReconciler(client, clientReader), - kserveRawISVCReconciler: reconcilers.NewKServeRawInferenceServiceReconciler(client), - } -} - -// Reconcile performs the reconciling of the Openshift objects for a Kubeflow -// InferenceService. -func (r *OpenshiftInferenceServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - // Initialize logger format - log := r.log.WithValues("InferenceService", req.Name, "namespace", req.Namespace) - // Get the InferenceService object when a reconciliation event is triggered (create, - // update, delete) - isvc := &kservev1beta1.InferenceService{} - err := r.client.Get(ctx, req.NamespacedName, isvc) - if err != nil && apierrs.IsNotFound(err) { - log.Info("Stop InferenceService reconciliation") - // InferenceService not found, so we check for any other inference services that might be using Kserve/ModelMesh - // If none are found, we delete the common namespace-scoped resources that were created for Kserve/ModelMesh. - err1 := r.DeleteResourcesIfNoIsvcExists(ctx, log, req.Namespace) - if err1 != nil { - log.Error(err1, "Unable to clean up resources") - return ctrl.Result{}, err1 - } - return ctrl.Result{}, nil - } else if err != nil { - log.Error(err, "Unable to fetch the InferenceService") - return ctrl.Result{}, err - } - - if isvc.GetDeletionTimestamp() != nil { - return reconcile.Result{}, r.onDeletion(ctx, log, isvc) - } - - // Check what deployment mode is used by the InferenceService. We have differing reconciliation logic for Kserve and ModelMesh - IsvcDeploymentMode, err := utils.GetDeploymentModeForIsvc(ctx, r.client, isvc) - if err != nil { - return ctrl.Result{}, err - } - switch IsvcDeploymentMode { - case constants.ModelMesh: - log.Info("Reconciling InferenceService for ModelMesh") - err = r.mmISVCReconciler.Reconcile(ctx, log, isvc) - case constants.Serverless: - log.Info("Reconciling InferenceService for Kserve in mode Serverless") - err = r.kserveServerlessISVCReconciler.Reconcile(ctx, log, isvc) - case constants.RawDeployment: - log.Info("Reconciling InferenceService for Kserve in mode RawDeployment") - err = r.kserveRawISVCReconciler.Reconcile(ctx, log, isvc) - } - - return ctrl.Result{}, err -} - -// SetupWithManager sets up the controller with the Manager. -func (r *OpenshiftInferenceServiceReconciler) SetupWithManager(mgr ctrl.Manager) error { - builder := ctrl.NewControllerManagedBy(mgr). - For(&kservev1beta1.InferenceService{}). - Owns(&kservev1alpha1.ServingRuntime{}). - Owns(&corev1.Namespace{}). - Owns(&routev1.Route{}). - Owns(&corev1.ServiceAccount{}). - Owns(&corev1.Service{}). - Owns(&corev1.ConfigMap{}). - Owns(&corev1.Secret{}). - Owns(&authv1.ClusterRoleBinding{}). - Owns(&networkingv1.NetworkPolicy{}). - Owns(&monitoringv1.ServiceMonitor{}). - Owns(&monitoringv1.PodMonitor{}). - Watches(&kservev1alpha1.ServingRuntime{}, - handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []reconcile.Request { - r.log.Info("Reconcile event triggered by serving runtime: " + o.GetName()) - inferenceServicesList := &kservev1beta1.InferenceServiceList{} - opts := []client.ListOption{client.InNamespace(o.GetNamespace())} - - // Todo: Get only Inference Services that are deploying on the specific serving runtime - err := r.client.List(ctx, inferenceServicesList, opts...) - if err != nil { - r.log.Info("Error getting list of inference services for namespace") - return []reconcile.Request{} - } - - if len(inferenceServicesList.Items) == 0 { - r.log.Info("No InferenceServices found for Serving Runtime: " + o.GetName()) - return []reconcile.Request{} - } - - reconcileRequests := make([]reconcile.Request, 0, len(inferenceServicesList.Items)) - for _, inferenceService := range inferenceServicesList.Items { - reconcileRequests = append(reconcileRequests, reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: inferenceService.Name, - Namespace: inferenceService.Namespace, - }, - }) - } - return reconcileRequests - })). - Watches(&corev1.Secret{}, - handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []reconcile.Request { - r.log.Info("Reconcile event triggered by Secret: " + o.GetName()) - isvc := &kservev1beta1.InferenceService{} - err := r.client.Get(ctx, types.NamespacedName{Name: o.GetName(), Namespace: o.GetNamespace()}, isvc) - if err != nil { - if apierrs.IsNotFound(err) { - return []reconcile.Request{} - } - r.log.Error(err, "Error getting the inferenceService", "name", o.GetName()) - return []reconcile.Request{} - } - - return []reconcile.Request{ - {NamespacedName: types.NamespacedName{Name: o.GetName(), Namespace: o.GetNamespace()}}, - } - })) - - kserveWithMeshEnabled, kserveWithMeshEnabledErr := utils.VerifyIfComponentIsEnabled(context.Background(), mgr.GetClient(), utils.KServeWithServiceMeshComponent) - if kserveWithMeshEnabledErr != nil { - r.log.V(1).Error(kserveWithMeshEnabledErr, "could not determine if kserve have service mesh enabled") - } - - isAuthConfigAvailable, crdErr := utils.IsCrdAvailable(mgr.GetConfig(), authorinov1beta2.GroupVersion.String(), "AuthConfig") - if crdErr != nil { - r.log.V(1).Error(crdErr, "could not determine if AuthConfig CRD is available") - return crdErr - } - - if kserveWithMeshEnabled && isAuthConfigAvailable { - r.log.Info("KServe is enabled and AuthConfig CRD is available, watching AuthConfigs") - builder.Owns(&authorinov1beta2.AuthConfig{}) - } else if kserveWithMeshEnabled { - r.log.Info("Using KServe with Service Mesh, but AuthConfig CRD is not installed - skipping AuthConfigs watches.") - } else { - r.log.Info("Didn't find KServe with Service Mesh.") - } - - return builder.Complete(r) -} - -// general clean-up, mostly resources in different namespaces from kservev1beta1.InferenceService -func (r *OpenshiftInferenceServiceReconciler) onDeletion(ctx context.Context, log logr.Logger, inferenceService *kservev1beta1.InferenceService) error { - log.V(1).Info("Running cleanup logic") - - IsvcDeploymentMode, err := utils.GetDeploymentModeForIsvc(ctx, r.client, inferenceService) - if err != nil { - log.V(1).Error(err, "Could not determine deployment mode for ISVC. Some resources related to the inferenceservice might not be deleted.") - } - if IsvcDeploymentMode == constants.Serverless { - log.V(1).Info("Deleting kserve inference resource (Serverless Mode)") - return r.kserveServerlessISVCReconciler.OnDeletionOfKserveInferenceService(ctx, log, inferenceService) - } - if IsvcDeploymentMode == constants.RawDeployment { - log.V(1).Info("Deleting kserve inference resource (RawDeployment Mode)") - return r.kserveRawISVCReconciler.OnDeletionOfKserveInferenceService(ctx, log, inferenceService) - } - return nil -} - -func (r *OpenshiftInferenceServiceReconciler) DeleteResourcesIfNoIsvcExists(ctx context.Context, log logr.Logger, namespace string) error { - if err := r.kserveServerlessISVCReconciler.CleanupNamespaceIfNoKserveIsvcExists(ctx, log, namespace); err != nil { - return err - } - if err := r.mmISVCReconciler.DeleteModelMeshResourcesIfNoMMIsvcExists(ctx, log, namespace); err != nil { - return err - } - if err := r.kserveRawISVCReconciler.CleanupNamespaceIfNoKserveIsvcExists(ctx, log, namespace); err != nil { - return err - } - return nil -} diff --git a/controllers/kserve_inferenceservice_controller_authconfig_test.go b/controllers/kserve_inferenceservice_controller_authconfig_test.go deleted file mode 100644 index 8dc775b7..00000000 --- a/controllers/kserve_inferenceservice_controller_authconfig_test.go +++ /dev/null @@ -1,377 +0,0 @@ -/* - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -import ( - "context" - kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - authorinov1beta2 "github.com/kuadrant/authorino/api/v1beta2" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/opendatahub-io/odh-model-controller/controllers/constants" - "github.com/opendatahub-io/odh-model-controller/controllers/utils" - istiosec_v1b1 "istio.io/client-go/pkg/apis/security/v1beta1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/dynamic" - "knative.dev/pkg/apis" -) - -var _ = Describe("InferenceService Authorization", func() { - - var ( - namespace *corev1.Namespace - isvc *kservev1beta1.InferenceService - ) - - When("not configured for the cluster", func() { - BeforeEach(func() { - ctx := context.Background() - namespace = &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: Namespaces.Get(), - }, - } - Expect(cli.Create(ctx, namespace)).Should(Succeed()) - inferenceServiceConfig := &corev1.ConfigMap{} - - Expect(convertToStructuredResource(InferenceServiceConfigPath1, inferenceServiceConfig)).To(Succeed()) - if err := cli.Create(ctx, inferenceServiceConfig); err != nil && !errors.IsAlreadyExists(err) { - Fail(err.Error()) - } - - // We need to stub the cluster state and indicate if Authorino is configured as authorization layer - if dsciErr := createDSCI(DSCIWithoutAuthorization); dsciErr != nil && !errors.IsAlreadyExists(dsciErr) { - Fail(dsciErr.Error()) - } - - isvc = createISVCWithoutAuth(namespace.Name) - }) - - AfterEach(func() { - Expect(deleteDSCI(DSCIWithoutAuthorization)).To(Succeed()) - }) - - It("should not create auth config", func() { - Consistently(func() error { - ac := &authorinov1beta2.AuthConfig{} - return getAuthConfig(namespace.Name, isvc.Name, ac) - }). - WithTimeout(timeout). - WithPolling(interval). - Should(Not(Succeed())) - }) - }) - - When("configured for the cluster", func() { - - BeforeEach(func() { - ctx := context.Background() - namespace = &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: Namespaces.Get(), - }, - } - Expect(cli.Create(ctx, namespace)).Should(Succeed()) - inferenceServiceConfig := &corev1.ConfigMap{} - - Expect(convertToStructuredResource(InferenceServiceConfigPath1, inferenceServiceConfig)).To(Succeed()) - if err := cli.Create(ctx, inferenceServiceConfig); err != nil && !errors.IsAlreadyExists(err) { - Fail(err.Error()) - } - - //// We need to stub the cluster state and indicate that Authorino is configured as authorization layer - //if dsciErr := createDSCI(DSCIWithAuthorization); dsciErr != nil && !errors.IsAlreadyExists(dsciErr) { - // Fail(dsciErr.Error()) - //} - - // TODO: See utils.VerifyIfMeshAuthorizationIsEnabled func - if authPolicyErr := createAuthorizationPolicy(KServeAuthorizationPolicy); authPolicyErr != nil && !errors.IsAlreadyExists(authPolicyErr) { - Fail(authPolicyErr.Error()) - } - }) - - AfterEach(func() { - //Expect(deleteDSCI(DSCIWithAuthorization)).To(Succeed()) - Expect(deleteAuthorizationPolicy(KServeAuthorizationPolicy)).To(Succeed()) - }) - - Context("when InferenceService is not ready", func() { - BeforeEach(func() { - isvc = createISVCMissingStatus(namespace.Name) - }) - - It("should not create auth config on missing status.URL", func() { - - Consistently(func() error { - ac := &authorinov1beta2.AuthConfig{} - return getAuthConfig(namespace.Name, isvc.Name, ac) - }). - WithTimeout(timeout). - WithPolling(interval). - Should(Not(Succeed())) - }) - }) - - Context("when InferenceService is ready", func() { - - Context("auth not enabled", func() { - BeforeEach(func() { - isvc = createISVCWithoutAuth(namespace.Name) - }) - - It("should create anonymous auth config", func() { - Expect(updateISVCStatus(isvc)).To(Succeed()) - - Eventually(func(g Gomega) { - ac := &authorinov1beta2.AuthConfig{} - g.Expect(getAuthConfig(namespace.Name, isvc.Name, ac)).To(Succeed()) - g.Expect(ac.Spec.Authorization["anonymous-access"]).NotTo(BeNil()) - }). - WithTimeout(timeout). - WithPolling(interval). - Should(Succeed()) - }) - - It("should update to non anonymous on enable", func() { - Expect(updateISVCStatus(isvc)).To(Succeed()) - - Eventually(func(g Gomega) { - ac := &authorinov1beta2.AuthConfig{} - g.Expect(getAuthConfig(namespace.Name, isvc.Name, ac)).To(Succeed()) - g.Expect(ac.Spec.Authorization["anonymous-access"]).NotTo(BeNil()) - }). - WithTimeout(timeout). - WithPolling(interval). - Should(Succeed()) - - Expect(enableAuth(isvc)).To(Succeed()) - Eventually(func(g Gomega) { - ac := &authorinov1beta2.AuthConfig{} - g.Expect(ac.Spec.Authorization["kubernetes-user"]).NotTo(BeNil()) - g.Expect(getAuthConfig(namespace.Name, isvc.Name, ac)).To(Succeed()) - }). - WithTimeout(timeout). - WithPolling(interval). - Should(Succeed()) - }) - }) - - Context("auth enabled", func() { - BeforeEach(func() { - isvc = createISVCWithAuth(namespace.Name) - }) - - It("should create user defined auth config", func() { - Expect(updateISVCStatus(isvc)).To(Succeed()) - - Eventually(func(g Gomega) { - ac := &authorinov1beta2.AuthConfig{} - g.Expect(getAuthConfig(namespace.Name, isvc.Name, ac)).To(Succeed()) - g.Expect(ac.Spec.Authorization["kubernetes-user"]).NotTo(BeNil()) - }). - WithTimeout(timeout). - WithPolling(interval). - Should(Succeed()) - }) - - It("should update to anonymous on disable", func() { - Expect(updateISVCStatus(isvc)).To(Succeed()) - - Eventually(func(g Gomega) { - ac := &authorinov1beta2.AuthConfig{} - g.Expect(getAuthConfig(namespace.Name, isvc.Name, ac)).To(Succeed()) - g.Expect(ac.Spec.Authorization["kubernetes-user"]).NotTo(BeNil()) - }). - WithTimeout(timeout). - WithPolling(interval). - Should(Succeed()) - - Expect(disableAuth(isvc)).To(Succeed()) - Eventually(func(g Gomega) { - ac := &authorinov1beta2.AuthConfig{} - g.Expect(getAuthConfig(namespace.Name, isvc.Name, ac)).To(Succeed()) - g.Expect(ac.Spec.Authorization["anonymous-access"]).NotTo(BeNil()) - }). - WithTimeout(timeout). - WithPolling(interval). - Should(Succeed()) - }) - }) - }) - }) - -}) - -func getAuthConfig(namespace, name string, ac *authorinov1beta2.AuthConfig) error { - return cli.Get(context.Background(), types.NamespacedName{Namespace: namespace, Name: name}, ac) -} - -func createISVCMissingStatus(namespace string) *kservev1beta1.InferenceService { - inferenceService := &kservev1beta1.InferenceService{} - err := convertToStructuredResource(KserveInferenceServicePath1, inferenceService) - Expect(err).NotTo(HaveOccurred()) - inferenceService.Namespace = namespace - Expect(cli.Create(ctx, inferenceService)).Should(Succeed()) - return inferenceService -} - -func createISVCWithAuth(namespace string) *kservev1beta1.InferenceService { - inferenceService := createBasicISVC(namespace) - inferenceService.Annotations[constants.LabelEnableAuth] = "true" - Expect(cli.Create(ctx, inferenceService)).Should(Succeed()) - - return inferenceService -} - -func createISVCWithoutAuth(namespace string) *kservev1beta1.InferenceService { - inferenceService := createBasicISVC(namespace) - Expect(cli.Create(ctx, inferenceService)).Should(Succeed()) - - return inferenceService -} - -func createBasicISVC(namespace string) *kservev1beta1.InferenceService { - inferenceService := &kservev1beta1.InferenceService{} - err := convertToStructuredResource(KserveInferenceServicePath1, inferenceService) - Expect(err).NotTo(HaveOccurred()) - inferenceService.Namespace = namespace - if inferenceService.Annotations == nil { - inferenceService.Annotations = map[string]string{} - } - return inferenceService -} - -func updateISVCStatus(isvc *kservev1beta1.InferenceService) error { - url, _ := apis.ParseURL("http://iscv-" + isvc.Namespace + "ns.apps.openshift.ai") - isvc.Status = kservev1beta1.InferenceServiceStatus{ - URL: url, - } - return cli.Status().Update(context.Background(), isvc) -} - -func disableAuth(isvc *kservev1beta1.InferenceService) error { - delete(isvc.Annotations, constants.LabelEnableAuth) - delete(isvc.Annotations, constants.LabelEnableAuthODH) - return cli.Update(context.Background(), isvc) -} - -func enableAuth(isvc *kservev1beta1.InferenceService) error { - if isvc.Annotations == nil { - isvc.Annotations = map[string]string{} - } - isvc.Annotations[constants.LabelEnableAuthODH] = "true" - return cli.Update(context.Background(), isvc) -} - -func createDSCI(dsci string) error { - obj := &unstructured.Unstructured{} - if err := convertToUnstructuredResource(dsci, obj); err != nil { - return err - } - - gvk := utils.GVK.DataScienceClusterInitialization - obj.SetGroupVersionKind(gvk) - dynamicClient, err := dynamic.NewForConfig(envTest.Config) - if err != nil { - return err - } - - gvr := schema.GroupVersionResource{ - Group: gvk.Group, - Version: gvk.Version, - Resource: "dscinitializations", - } - resource := dynamicClient.Resource(gvr) - createdObj, createErr := resource.Create(context.TODO(), obj, metav1.CreateOptions{}) - if createErr != nil { - return nil - } - - if status, found, err := unstructured.NestedFieldCopy(obj.Object, "status"); err != nil { - return err - } else if found { - if err := unstructured.SetNestedField(createdObj.Object, status, "status"); err != nil { - return err - } - } - - _, statusErr := resource.UpdateStatus(context.TODO(), createdObj, metav1.UpdateOptions{}) - - return statusErr -} - -func createAuthorizationPolicy(authPolicyFile string) error { - obj := &unstructured.Unstructured{} - if err := convertToUnstructuredResource(authPolicyFile, obj); err != nil { - return err - } - - obj.SetGroupVersionKind(istiosec_v1b1.SchemeGroupVersion.WithKind("AuthorizationPolicy")) - dynamicClient, err := dynamic.NewForConfig(envTest.Config) - if err != nil { - return err - } - - gvr := istiosec_v1b1.SchemeGroupVersion.WithResource("authorizationpolicies") - resource := dynamicClient.Resource(gvr) - _, meshNamespace := utils.GetIstioControlPlaneName(ctx, cli) - _, createErr := resource.Namespace(meshNamespace).Create(context.TODO(), obj, metav1.CreateOptions{}) - - return createErr -} - -func deleteDSCI(dsci string) error { - obj := &unstructured.Unstructured{} - if err := convertToUnstructuredResource(dsci, obj); err != nil { - return err - } - - gvk := utils.GVK.DataScienceClusterInitialization - obj.SetGroupVersionKind(gvk) - dynamicClient, err := dynamic.NewForConfig(envTest.Config) - if err != nil { - return err - } - - gvr := schema.GroupVersionResource{ - Group: gvk.Group, - Version: gvk.Version, - Resource: "dscinitializations", - } - return dynamicClient.Resource(gvr).Delete(context.TODO(), obj.GetName(), metav1.DeleteOptions{}) -} - -func deleteAuthorizationPolicy(authPolicyFile string) error { - obj := &unstructured.Unstructured{} - if err := convertToUnstructuredResource(authPolicyFile, obj); err != nil { - return err - } - - obj.SetGroupVersionKind(istiosec_v1b1.SchemeGroupVersion.WithKind("AuthorizationPolicy")) - dynamicClient, err := dynamic.NewForConfig(envTest.Config) - if err != nil { - return err - } - - gvr := istiosec_v1b1.SchemeGroupVersion.WithResource("authorizationpolicies") - _, meshNamespace := utils.GetIstioControlPlaneName(ctx, cli) - return dynamicClient.Resource(gvr).Namespace(meshNamespace).Delete(context.TODO(), obj.GetName(), metav1.DeleteOptions{}) -} diff --git a/controllers/kserve_inferenceservice_controller_mesh_test.go b/controllers/kserve_inferenceservice_controller_mesh_test.go deleted file mode 100644 index 720be204..00000000 --- a/controllers/kserve_inferenceservice_controller_mesh_test.go +++ /dev/null @@ -1,166 +0,0 @@ -package controllers - -import ( - "reflect" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - maistrav1 "maistra.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/opendatahub-io/odh-model-controller/controllers/constants" - "github.com/opendatahub-io/odh-model-controller/controllers/utils" -) - -var _ = Describe("The KServe mesh reconciler", func() { - var testNs string - - createInferenceService := func(namespace, name string) *kservev1beta1.InferenceService { - inferenceService := &kservev1beta1.InferenceService{} - err := convertToStructuredResource(KserveInferenceServicePath1, inferenceService) - Expect(err).NotTo(HaveOccurred()) - inferenceService.SetNamespace(namespace) - if len(name) != 0 { - inferenceService.Name = name - } - Expect(cli.Create(ctx, inferenceService)).Should(Succeed()) - - return inferenceService - } - - expectOwnedSmmCreated := func(namespace string) { - Eventually(func() error { - smm := &maistrav1.ServiceMeshMember{} - key := types.NamespacedName{Name: constants.ServiceMeshMemberName, Namespace: namespace} - err := cli.Get(ctx, key, smm) - return err - }, timeout, interval).Should(Succeed()) - } - - createUserOwnedMeshEnrolment := func(namespace string) *maistrav1.ServiceMeshMember { - controlPlaneName, meshNamespace := utils.GetIstioControlPlaneName(ctx, cli) - smm := &maistrav1.ServiceMeshMember{ - ObjectMeta: metav1.ObjectMeta{ - Name: constants.ServiceMeshMemberName, - Namespace: namespace, - Labels: nil, - Annotations: nil, - }, - Spec: maistrav1.ServiceMeshMemberSpec{ - ControlPlaneRef: maistrav1.ServiceMeshControlPlaneRef{ - Name: controlPlaneName, - Namespace: meshNamespace, - }}, - Status: maistrav1.ServiceMeshMemberStatus{}, - } - Expect(cli.Create(ctx, smm)).Should(Succeed()) - - return smm - } - - BeforeEach(func() { - testNamespace := Namespaces.Create(cli) - testNs = testNamespace.Name - - inferenceServiceConfig := &corev1.ConfigMap{} - Expect(convertToStructuredResource(InferenceServiceConfigPath1, inferenceServiceConfig)).To(Succeed()) - if err := cli.Create(ctx, inferenceServiceConfig); err != nil && !errors.IsAlreadyExists(err) { - Fail(err.Error()) - } - }) - - When("deploying the first model in a namespace", func() { - It("if the namespace is not part of the service mesh, it should enroll the namespace to the mesh", func() { - inferenceService := createInferenceService(testNs, "") - expectOwnedSmmCreated(inferenceService.Namespace) - }) - - It("if the namespace is already enrolled to the service mesh by the user, it should not modify the enrollment", func() { - smm := createUserOwnedMeshEnrolment(testNs) - inferenceService := createInferenceService(testNs, "") - - Consistently(func() bool { - actualSmm := &maistrav1.ServiceMeshMember{} - key := types.NamespacedName{Name: constants.ServiceMeshMemberName, Namespace: inferenceService.Namespace} - err := cli.Get(ctx, key, actualSmm) - return err == nil && reflect.DeepEqual(actualSmm, smm) - }).Should(BeTrue()) - }) - - It("if the namespace is already enrolled to some other control plane, it should anyway not modify the enrollment", func() { - smm := &maistrav1.ServiceMeshMember{ - ObjectMeta: metav1.ObjectMeta{ - Name: constants.ServiceMeshMemberName, - Namespace: testNs, - Labels: nil, - Annotations: nil, - }, - Spec: maistrav1.ServiceMeshMemberSpec{ - ControlPlaneRef: maistrav1.ServiceMeshControlPlaneRef{ - Name: "random-control-plane-vbfr238497", - Namespace: "random-namespace-a234h", - }}, - Status: maistrav1.ServiceMeshMemberStatus{}, - } - Expect(cli.Create(ctx, smm)).Should(Succeed()) - - inferenceService := createInferenceService(testNs, "") - - Consistently(func() bool { - actualSmm := &maistrav1.ServiceMeshMember{} - key := types.NamespacedName{Name: constants.ServiceMeshMemberName, Namespace: inferenceService.Namespace} - err := cli.Get(ctx, key, actualSmm) - return err == nil && reflect.DeepEqual(actualSmm, smm) - }).Should(BeTrue()) - }) - }) - - When("deleting the last model in a namespace", func() { - It("it should remove the owned service mesh enrolment", func() { - inferenceService := createInferenceService(testNs, "") - expectOwnedSmmCreated(inferenceService.Namespace) - - Expect(cli.Delete(ctx, inferenceService)).Should(Succeed()) - Eventually(func() error { - smm := &maistrav1.ServiceMeshMember{} - key := types.NamespacedName{Name: constants.ServiceMeshMemberName, Namespace: inferenceService.Namespace} - err := cli.Get(ctx, key, smm) - return err - }, timeout, interval).ShouldNot(Succeed()) - }) - - It("it should not remove a user-owned service mesh enrolment", func() { - createUserOwnedMeshEnrolment(testNs) - inferenceService := createInferenceService(testNs, "") - - Expect(cli.Delete(ctx, inferenceService)).Should(Succeed()) - Consistently(func() int { - smmList := &maistrav1.ServiceMeshMemberList{} - Expect(cli.List(ctx, smmList, client.InNamespace(inferenceService.Namespace))).Should(Succeed()) - return len(smmList.Items) - }).Should(Equal(1)) - }) - }) - - When("deleting a model, but there are other models left in the namespace", func() { - It("it should not remove the owned service mesh enrolment", func() { - inferenceService1 := createInferenceService(testNs, "") - createInferenceService(testNs, "secondary-isvc") - expectOwnedSmmCreated(inferenceService1.Namespace) - - Expect(cli.Delete(ctx, inferenceService1)).Should(Succeed()) - Consistently(func() error { - smm := &maistrav1.ServiceMeshMember{} - key := types.NamespacedName{Name: constants.ServiceMeshMemberName, Namespace: inferenceService1.Namespace} - err := cli.Get(ctx, key, smm) - return err - }, timeout, interval).Should(Succeed()) - }) - }) -}) diff --git a/controllers/kserve_inferenceservice_controller_metrics_test.go b/controllers/kserve_inferenceservice_controller_metrics_test.go deleted file mode 100644 index c40c4096..00000000 --- a/controllers/kserve_inferenceservice_controller_metrics_test.go +++ /dev/null @@ -1,176 +0,0 @@ -package controllers - -import ( - "github.com/opendatahub-io/odh-model-controller/controllers/constants" - "github.com/opendatahub-io/odh-model-controller/controllers/utils" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "time" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - kservev1alpha1 "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" - kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" -) - -const ( - KserveOvmsInferenceServiceName = "example-onnx-mnist" - UnsupportedMetricsInferenceServiceName = "sklearn-v2-iris" - NilRuntimeInferenceServiceName = "sklearn-v2-iris-no-runtime" - NilModelInferenceServiceName = "custom-runtime" - UnsupportedMetricsInferenceServicePath = "./testdata/deploy/kserve-unsupported-metrics-inference-service.yaml" - UnsupprtedMetricsServingRuntimePath = "./testdata/deploy/kserve-unsupported-metrics-serving-runtime.yaml" - NilRuntimeInferenceServicePath = "./testdata/deploy/kserve-nil-runtime-inference-service.yaml" - NilModelInferenceServicePath = "./testdata/deploy/kserve-nil-model-inference-service.yaml" -) - -var _ = Describe("The KServe Dashboard reconciler", func() { - var testNs string - - createServingRuntime := func(namespace, path string) *kservev1alpha1.ServingRuntime { - servingRuntime := &kservev1alpha1.ServingRuntime{} - err := convertToStructuredResource(path, servingRuntime) - Expect(err).NotTo(HaveOccurred()) - servingRuntime.SetNamespace(namespace) - if err := cli.Create(ctx, servingRuntime); err != nil && !errors.IsAlreadyExists(err) { - Fail(err.Error()) - } - return servingRuntime - } - - createInferenceService := func(namespace, name string, path string) *kservev1beta1.InferenceService { - inferenceService := &kservev1beta1.InferenceService{} - err := convertToStructuredResource(path, inferenceService) - Expect(err).NotTo(HaveOccurred()) - inferenceService.SetNamespace(namespace) - if len(name) != 0 { - inferenceService.Name = name - } - if err := cli.Create(ctx, inferenceService); err != nil && !errors.IsAlreadyExists(err) { - Fail(err.Error()) - } - return inferenceService - } - - BeforeEach(func() { - testNs = Namespaces.Create(cli).Name - - inferenceServiceConfig := &corev1.ConfigMap{} - Expect(convertToStructuredResource(InferenceServiceConfigPath1, inferenceServiceConfig)).To(Succeed()) - if err := cli.Create(ctx, inferenceServiceConfig); err != nil && !errors.IsAlreadyExists(err) { - Fail(err.Error()) - } - }) - - When("deploying a Kserve model", func() { - It("if the runtime is supported for metrics, it should create a configmap with prometheus queries", func() { - _ = createServingRuntime(testNs, KserveServingRuntimePath1) - _ = createInferenceService(testNs, KserveOvmsInferenceServiceName, KserveInferenceServicePath1) - - metricsConfigMap, err := waitForConfigMap(cli, testNs, KserveOvmsInferenceServiceName+constants.KserveMetricsConfigMapNameSuffix, 30, 1*time.Second) - Expect(err).NotTo(HaveOccurred()) - Expect(metricsConfigMap).NotTo(BeNil()) - - finaldata := utils.SubstituteVariablesInQueries(constants.OvmsMetricsData, testNs, KserveOvmsInferenceServiceName) - expectedMetricsConfigMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: KserveOvmsInferenceServiceName + constants.KserveMetricsConfigMapNameSuffix, - Namespace: testNs, - }, - Data: map[string]string{ - "supported": "true", - "metrics": finaldata, - }, - } - Expect(compareConfigMap(metricsConfigMap, expectedMetricsConfigMap)).Should(BeTrue()) - Expect(expectedMetricsConfigMap.Data).NotTo(HaveKeyWithValue("metrics", ContainSubstring("${REQUEST_RATE_INTERVAL}"))) - }) - - It("if the runtime is not supported for metrics, it should create a configmap with the unsupported config", func() { - _ = createServingRuntime(testNs, UnsupprtedMetricsServingRuntimePath) - _ = createInferenceService(testNs, UnsupportedMetricsInferenceServiceName, UnsupportedMetricsInferenceServicePath) - - metricsConfigMap, err := waitForConfigMap(cli, testNs, UnsupportedMetricsInferenceServiceName+constants.KserveMetricsConfigMapNameSuffix, 30, 1*time.Second) - Expect(err).NotTo(HaveOccurred()) - Expect(metricsConfigMap).NotTo(BeNil()) - - expectedMetricsConfigMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: UnsupportedMetricsInferenceServiceName + constants.KserveMetricsConfigMapNameSuffix, - Namespace: testNs, - }, - Data: map[string]string{ - "supported": "false", - }, - } - Expect(compareConfigMap(metricsConfigMap, expectedMetricsConfigMap)).Should(BeTrue()) - }) - - It("if the isvc does not have a runtime specified, an unsupported metrics configmap should be created", func() { - _ = createInferenceService(testNs, NilRuntimeInferenceServiceName, NilRuntimeInferenceServicePath) - - metricsConfigMap, err := waitForConfigMap(cli, testNs, NilRuntimeInferenceServiceName+constants.KserveMetricsConfigMapNameSuffix, 30, 1*time.Second) - Expect(err).NotTo(HaveOccurred()) - Expect(metricsConfigMap).NotTo(BeNil()) - - expectedmetricsConfigMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: NilRuntimeInferenceServiceName + constants.KserveMetricsConfigMapNameSuffix, - Namespace: testNs, - }, - Data: map[string]string{ - "supported": "false", - }, - } - Expect(compareConfigMap(metricsConfigMap, expectedmetricsConfigMap)).Should(BeTrue()) - }) - - It("if the isvc does not have the model field specified, an unsupported metrics configmap should be created", func() { - _ = createInferenceService(testNs, NilModelInferenceServiceName, NilModelInferenceServicePath) - - metricsConfigMap, err := waitForConfigMap(cli, testNs, NilModelInferenceServiceName+constants.KserveMetricsConfigMapNameSuffix, 30, 1*time.Second) - Expect(err).NotTo(HaveOccurred()) - Expect(metricsConfigMap).NotTo(BeNil()) - - expectedCM := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: NilModelInferenceServiceName + constants.KserveMetricsConfigMapNameSuffix, - Namespace: testNs, - }, - Data: map[string]string{ - "supported": "false", - }, - } - Expect(compareConfigMap(metricsConfigMap, expectedCM)).Should(BeTrue()) - }) - }) - - When("deleting the deployed models", func() { - It("it should delete the associated configmap", func() { - _ = createServingRuntime(testNs, KserveServingRuntimePath1) - OvmsInferenceService := createInferenceService(testNs, KserveOvmsInferenceServiceName, KserveInferenceServicePath1) - - Expect(cli.Delete(ctx, OvmsInferenceService)).Should(Succeed()) - Eventually(func() error { - configmap := &corev1.ConfigMap{} - key := types.NamespacedName{Name: KserveOvmsInferenceServiceName + constants.KserveMetricsConfigMapNameSuffix, Namespace: OvmsInferenceService.Namespace} - err := cli.Get(ctx, key, configmap) - return err - }, timeout, interval).ShouldNot(Succeed()) - - _ = createServingRuntime(testNs, UnsupprtedMetricsServingRuntimePath) - SklearnInferenceService := createInferenceService(testNs, UnsupportedMetricsInferenceServiceName, UnsupportedMetricsInferenceServicePath) - - Expect(cli.Delete(ctx, SklearnInferenceService)).Should(Succeed()) - Eventually(func() error { - configmap := &corev1.ConfigMap{} - key := types.NamespacedName{Name: UnsupportedMetricsInferenceServiceName + constants.KserveMetricsConfigMapNameSuffix, Namespace: SklearnInferenceService.Namespace} - err := cli.Get(ctx, key, configmap) - return err - }, timeout, interval).ShouldNot(Succeed()) - }) - }) -}) diff --git a/controllers/kserve_inferenceservice_controller_test.go b/controllers/kserve_inferenceservice_controller_test.go deleted file mode 100644 index b1f94451..00000000 --- a/controllers/kserve_inferenceservice_controller_test.go +++ /dev/null @@ -1,464 +0,0 @@ -/* - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -import ( - "context" - "fmt" - - "strings" - - kservev1alpha1 "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" - kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - . "github.com/onsi/gomega/gstruct" - gomegatypes "github.com/onsi/gomega/types" - "github.com/opendatahub-io/odh-model-controller/controllers/constants" - "github.com/opendatahub-io/odh-model-controller/controllers/utils" - routev1 "github.com/openshift/api/route/v1" - istioclientv1beta1 "istio.io/client-go/pkg/apis/networking/v1beta1" - corev1 "k8s.io/api/core/v1" - v1 "k8s.io/api/networking/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - "knative.dev/pkg/apis" - duckv1 "knative.dev/pkg/apis/duck/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const ( - testIsvcSvcPath = "./testdata/servingcert-service/test-isvc-svc.yaml" - kserveLocalGatewayPath = "./testdata/gateway/kserve-local-gateway.yaml" - testIsvcSvcSecretPath = "./testdata/gateway/test-isvc-svc-secret.yaml" -) - -var _ = Describe("The Openshift Kserve model controller", func() { - - When("creating a Kserve ServiceRuntime & InferenceService", func() { - var testNs string - - BeforeEach(func() { - ctx := context.Background() - testNamespace := Namespaces.Create(cli) - testNs = testNamespace.Name - - inferenceServiceConfig := &corev1.ConfigMap{} - Expect(convertToStructuredResource(InferenceServiceConfigPath1, inferenceServiceConfig)).To(Succeed()) - if err := cli.Create(ctx, inferenceServiceConfig); err != nil && !errors.IsAlreadyExists(err) { - Fail(err.Error()) - } - - servingRuntime := &kservev1alpha1.ServingRuntime{} - Expect(convertToStructuredResource(KserveServingRuntimePath1, servingRuntime)).To(Succeed()) - if err := cli.Create(ctx, servingRuntime); err != nil && !errors.IsAlreadyExists(err) { - Fail(err.Error()) - } - }) - - It("With Kserve InferenceService a Route be created", func() { - _, meshNamespace := utils.GetIstioControlPlaneName(ctx, cli) - inferenceService := &kservev1beta1.InferenceService{} - err := convertToStructuredResource(KserveInferenceServicePath1, inferenceService) - Expect(err).NotTo(HaveOccurred()) - inferenceService.SetNamespace(testNs) - - Expect(cli.Create(ctx, inferenceService)).Should(Succeed()) - - By("By checking that the controller has not created the Route") - Consistently(func() error { - route := &routev1.Route{} - key := types.NamespacedName{Name: getKServeRouteName(inferenceService), Namespace: meshNamespace} - err = cli.Get(ctx, key, route) - return err - }, timeout, interval).Should(HaveOccurred()) - - deployedInferenceService := &kservev1beta1.InferenceService{} - err = cli.Get(ctx, types.NamespacedName{Name: inferenceService.Name, Namespace: inferenceService.Namespace}, deployedInferenceService) - Expect(err).NotTo(HaveOccurred()) - - url, err := apis.ParseURL("https://example-onnx-mnist-default.test.com") - Expect(err).NotTo(HaveOccurred()) - deployedInferenceService.Status.URL = url - - err = cli.Status().Update(ctx, deployedInferenceService) - Expect(err).NotTo(HaveOccurred()) - - By("By checking that the controller has created the Route") - Eventually(func() error { - route := &routev1.Route{} - key := types.NamespacedName{Name: getKServeRouteName(inferenceService), Namespace: meshNamespace} - err = cli.Get(ctx, key, route) - return err - }, timeout, interval).Should(Succeed()) - }) - It("With a new Kserve InferenceService, serving cert annotation should be added to the runtime Service object.", func() { - // We need to stub the cluster state and indicate where is istio namespace (reusing authConfig test data) - if dsciErr := createDSCI(DSCIWithoutAuthorization); dsciErr != nil && !errors.IsAlreadyExists(dsciErr) { - Fail(dsciErr.Error()) - } - // Create a new InferenceService - inferenceService := &kservev1beta1.InferenceService{} - err := convertToStructuredResource(KserveInferenceServicePath1, inferenceService) - Expect(err).NotTo(HaveOccurred()) - inferenceService.SetNamespace(testNs) - Expect(cli.Create(ctx, inferenceService)).Should(Succeed()) - // Update the URL of the InferenceService to indicate it is ready. - deployedInferenceService := &kservev1beta1.InferenceService{} - err = cli.Get(ctx, types.NamespacedName{Name: inferenceService.Name, Namespace: inferenceService.Namespace}, deployedInferenceService) - Expect(err).NotTo(HaveOccurred()) - // url, err := apis.ParseURL("https://example-onnx-mnist-default.test.com") - Expect(err).NotTo(HaveOccurred()) - newAddress := &duckv1.Addressable{ - URL: apis.HTTPS("example-onnx-mnist-default.test.com"), - } - deployedInferenceService.Status.Address = newAddress - err = cli.Status().Update(ctx, deployedInferenceService) - Expect(err).NotTo(HaveOccurred()) - // Stub: Create a Kserve Service, which must be created by the KServe operator. - svc := &corev1.Service{} - err = convertToStructuredResource(testIsvcSvcPath, svc) - Expect(err).NotTo(HaveOccurred()) - svc.SetNamespace(inferenceService.Namespace) - Expect(cli.Create(ctx, svc)).Should(Succeed()) - err = cli.Status().Update(ctx, deployedInferenceService) - Expect(err).NotTo(HaveOccurred()) - // isvcService, err := waitForService(cli, testNs, inferenceService.Name, 5, 2*time.Second) - // Expect(err).NotTo(HaveOccurred()) - - isvcService := &corev1.Service{} - Eventually(func() error { - err := cli.Get(ctx, client.ObjectKey{Namespace: inferenceService.Namespace, Name: inferenceService.Name}, isvcService) - if err != nil { - return err - } - if isvcService.Annotations == nil || isvcService.Annotations[constants.ServingCertAnnotationKey] == "" { - - return fmt.Errorf("Annotation[constants.ServingCertAnnotationKey] is not added yet") - } - return nil - }, timeout, interval).Should(Succeed()) - - Expect(isvcService.Annotations[constants.ServingCertAnnotationKey]).Should(Equal(inferenceService.Name)) - }) - - It("should create a secret for runtime and update kserve local gateway in the istio-system namespace", func() { - // We need to stub the cluster state and indicate where is istio namespace (reusing authConfig test data) - if dsciErr := createDSCI(DSCIWithoutAuthorization); dsciErr != nil && !errors.IsAlreadyExists(dsciErr) { - Fail(dsciErr.Error()) - } - // Stub: Create a kserve-local-gateway, which must be created by the OpenDataHub operator. - kserveLocalGateway := &istioclientv1beta1.Gateway{} - err := convertToStructuredResource(kserveLocalGatewayPath, kserveLocalGateway) - Expect(err).NotTo(HaveOccurred()) - Expect(cli.Create(ctx, kserveLocalGateway)).Should(Succeed()) - - // Stub: Create a certificate Secret, which must be created by the openshift service-ca operator. - secret := &corev1.Secret{} - err = convertToStructuredResource(testIsvcSvcSecretPath, secret) - Expect(err).NotTo(HaveOccurred()) - secret.SetNamespace(testNs) - Expect(cli.Create(ctx, secret)).Should(Succeed()) - - // Create a new InferenceService - inferenceService := &kservev1beta1.InferenceService{} - err = convertToStructuredResource(KserveInferenceServicePath1, inferenceService) - Expect(err).NotTo(HaveOccurred()) - inferenceService.SetNamespace(testNs) - - Expect(cli.Create(ctx, inferenceService)).Should(Succeed()) - - // Update the URL of the InferenceService to indicate it is ready. - deployedInferenceService := &kservev1beta1.InferenceService{} - err = cli.Get(ctx, types.NamespacedName{Name: inferenceService.Name, Namespace: inferenceService.Namespace}, deployedInferenceService) - Expect(err).NotTo(HaveOccurred()) - - newAddress := &duckv1.Addressable{ - URL: apis.HTTPS("example-onnx-mnist-default.test.com"), - } - deployedInferenceService.Status.Address = newAddress - - err = cli.Status().Update(ctx, deployedInferenceService) - Expect(err).NotTo(HaveOccurred()) - - _, meshNamespace := utils.GetIstioControlPlaneName(ctx, cli) - - // Verify that the certificate secret is created in the istio-system namespace. - Eventually(func() error { - secret := &corev1.Secret{} - return cli.Get(ctx, client.ObjectKey{Namespace: meshNamespace, Name: fmt.Sprintf("%s-%s", inferenceService.Name, inferenceService.Namespace)}, secret) - }, timeout, interval).Should(Succeed()) - - // Verify that the gateway is updated in the istio-system namespace. - var gateway *istioclientv1beta1.Gateway - Eventually(func() error { - gateway, err = waitForUpdatedGatewayCompletion(cli, "add", meshNamespace, constants.KServeGatewayName, inferenceService.Name) - return err - }, timeout, interval).Should(Succeed()) - - // Ensure that the server is successfully added to the KServe local gateway within the istio-system namespace. - targetServerExist := hasServerFromGateway(gateway, fmt.Sprintf("%s-%s", "https", inferenceService.Name)) - Expect(targetServerExist).Should(BeTrue()) - }) - - It("should create required network policies when KServe is used", func() { - // given - inferenceService := &kservev1beta1.InferenceService{} - Expect(convertToStructuredResource(KserveInferenceServicePath1, inferenceService)).To(Succeed()) - inferenceService.SetNamespace(testNs) - - // when - Expect(cli.Create(ctx, inferenceService)).Should(Succeed()) - - // then - By("ensuring that the controller has created required network policies") - networkPolicies := &v1.NetworkPolicyList{} - Eventually(func() []v1.NetworkPolicy { - err := cli.List(ctx, networkPolicies, client.InNamespace(inferenceService.Namespace)) - if err != nil { - Fail(err.Error()) - } - return networkPolicies.Items - }, timeout, interval).Should( - ContainElements( - withMatchingNestedField("ObjectMeta.Name", Equal("allow-from-openshift-monitoring-ns")), - withMatchingNestedField("ObjectMeta.Name", Equal("allow-openshift-ingress")), - withMatchingNestedField("ObjectMeta.Name", Equal("allow-from-opendatahub-ns")), - ), - ) - }) - }) - - Context("when there is a existing inferenceService", func() { - var testNs string - var isvcName string - - BeforeEach(func() { - ctx := context.Background() - testNamespace := Namespaces.Create(cli) - testNs = testNamespace.Name - _, meshNamespace := utils.GetIstioControlPlaneName(ctx, cli) - - inferenceServiceConfig := &corev1.ConfigMap{} - Expect(convertToStructuredResource(InferenceServiceConfigPath1, inferenceServiceConfig)).To(Succeed()) - if err := cli.Create(ctx, inferenceServiceConfig); err != nil && !errors.IsAlreadyExists(err) { - Fail(err.Error()) - } - - // We need to stub the cluster state and indicate where is istio namespace (reusing authConfig test data) - if dsciErr := createDSCI(DSCIWithoutAuthorization); dsciErr != nil && !errors.IsAlreadyExists(dsciErr) { - Fail(dsciErr.Error()) - } - - servingRuntime := &kservev1alpha1.ServingRuntime{} - Expect(convertToStructuredResource(KserveServingRuntimePath1, servingRuntime)).To(Succeed()) - if err := cli.Create(ctx, servingRuntime); err != nil && !errors.IsAlreadyExists(err) { - Fail(err.Error()) - } - - // Stub: Create a kserve-local-gateway, which must be created by the OpenDataHub operator. - kserveLocalGateway := &istioclientv1beta1.Gateway{} - err := convertToStructuredResource(kserveLocalGatewayPath, kserveLocalGateway) - Expect(err).NotTo(HaveOccurred()) - Expect(cli.Create(ctx, kserveLocalGateway)).Should(Succeed()) - - // Stub: Create a certificate Secret, which must be created by the openshift service-ca operator. - secret := &corev1.Secret{} - err = convertToStructuredResource(testIsvcSvcSecretPath, secret) - Expect(err).NotTo(HaveOccurred()) - secret.SetNamespace(testNs) - Expect(cli.Create(ctx, secret)).Should(Succeed()) - - // Create a new InferenceService - inferenceService := &kservev1beta1.InferenceService{} - err = convertToStructuredResource(KserveInferenceServicePath1, inferenceService) - Expect(err).NotTo(HaveOccurred()) - inferenceService.SetNamespace(testNs) - // Ensure the Delete method is called when the InferenceService (ISVC) is deleted. - inferenceService.SetFinalizers([]string{"finalizer.inferenceservice"}) - - Expect(cli.Create(ctx, inferenceService)).Should(Succeed()) - isvcName = inferenceService.Name - - // Update the URL of the InferenceService to indicate it is ready. - deployedInferenceService := &kservev1beta1.InferenceService{} - err = cli.Get(ctx, types.NamespacedName{Name: inferenceService.Name, Namespace: testNs}, deployedInferenceService) - Expect(err).NotTo(HaveOccurred()) - - newAddress := &duckv1.Addressable{ - URL: apis.HTTPS("example-onnx-mnist-default.test.com"), - } - deployedInferenceService.Status.Address = newAddress - - err = cli.Status().Update(ctx, deployedInferenceService) - Expect(err).NotTo(HaveOccurred()) - - // Verify that the certificate secret is created in the istio-system namespace. - Eventually(func() error { - secret := &corev1.Secret{} - return cli.Get(ctx, types.NamespacedName{Name: inferenceService.Name, Namespace: inferenceService.Namespace}, secret) - }, timeout, interval).Should(Succeed()) - - Eventually(func() error { - return cli.Get(ctx, client.ObjectKey{Namespace: meshNamespace, Name: fmt.Sprintf("%s-%s", inferenceService.Name, inferenceService.Namespace)}, secret) - }, timeout, interval).Should(Succeed()) - - // Verify that the gateway is updated in the istio-system namespace. - var gateway *istioclientv1beta1.Gateway - Eventually(func() error { - gateway, err = waitForUpdatedGatewayCompletion(cli, "add", meshNamespace, constants.KServeGatewayName, inferenceService.Name) - return err - }, timeout, interval).Should(Succeed()) - - // Ensure that the server is successfully added to the KServe local gateway within the istio-system namespace. - targetServerExist := hasServerFromGateway(gateway, fmt.Sprintf("%s-%s", "https", inferenceService.Name)) - Expect(targetServerExist).Should(BeTrue()) - }) - - When("serving cert Secret is rotated", func() { - It("should re-sync serving cert Secret to istio-system", func() { - deployedInferenceService := &kservev1beta1.InferenceService{} - err := cli.Get(ctx, types.NamespacedName{Name: isvcName, Namespace: testNs}, deployedInferenceService) - Expect(err).NotTo(HaveOccurred()) - - // Get source secret - srcSecret := &corev1.Secret{} - err = cli.Get(ctx, client.ObjectKey{Namespace: testNs, Name: deployedInferenceService.Name}, srcSecret) - Expect(err).NotTo(HaveOccurred()) - - // Update source secret - updatedDataString := "updateData" - srcSecret.Data["tls.crt"] = []byte(updatedDataString) - srcSecret.Data["tls.key"] = []byte(updatedDataString) - Expect(cli.Update(ctx, srcSecret)).Should(Succeed()) - - // Get destination secret - err = cli.Get(ctx, client.ObjectKey{Namespace: testNs, Name: deployedInferenceService.Name}, srcSecret) - Expect(err).NotTo(HaveOccurred()) - - // Verify that the certificate secret in the istio-system namespace is updated. - _, meshNamespace := utils.GetIstioControlPlaneName(ctx, cli) - destSecret := &corev1.Secret{} - Eventually(func() error { - Expect(cli.Get(ctx, client.ObjectKey{Namespace: meshNamespace, Name: fmt.Sprintf("%s-%s", deployedInferenceService.Name, deployedInferenceService.Namespace)}, destSecret)).Should(Succeed()) - if string(destSecret.Data["tls.crt"]) != updatedDataString { - return fmt.Errorf("destSecret is not updated yet") - } - return nil - }, timeout, interval).Should(Succeed()) - - Expect(destSecret.Data).To(Equal(srcSecret.Data)) - }) - }) - - When("infereceService is deleted", func() { - It("should remove the Server from the kserve local gateway in istio-system and delete the created Secret", func() { - // Delete the existing ISVC - deployedInferenceService := &kservev1beta1.InferenceService{} - err := cli.Get(ctx, types.NamespacedName{Name: isvcName, Namespace: testNs}, deployedInferenceService) - Expect(err).NotTo(HaveOccurred()) - Expect(cli.Delete(ctx, deployedInferenceService)).Should(Succeed()) - - _, meshNamespace := utils.GetIstioControlPlaneName(ctx, cli) - - // Verify that the gateway is updated in the istio-system namespace. - var gateway *istioclientv1beta1.Gateway - Eventually(func() error { - gateway, err = waitForUpdatedGatewayCompletion(cli, "delete", meshNamespace, constants.KServeGatewayName, isvcName) - return err - }, timeout, interval).Should(Succeed()) - - // Ensure that the server is successfully removed from the KServe local gateway within the istio-system namespace. - targetServerExist := hasServerFromGateway(gateway, isvcName) - Expect(targetServerExist).Should(BeFalse()) - - // Ensure that the synced Secret is successfully deleted within the istio-system namespace. - secret := &corev1.Secret{} - Eventually(func() error { - return cli.Get(ctx, client.ObjectKey{Namespace: meshNamespace, Name: fmt.Sprintf("%s-%s", isvcName, meshNamespace)}, secret) - }, timeout, interval).ShouldNot(Succeed()) - }) - }) - }) - -}) - -func withMatchingNestedField(path string, matcher gomegatypes.GomegaMatcher) gomegatypes.GomegaMatcher { - if path == "" { - Fail("cannot handle empty path") - } - - fields := strings.Split(path, ".") - - // Reverse the path, so we start composing matchers from the leaf up - for i, j := 0, len(fields)-1; i < j; i, j = i+1, j-1 { - fields[i], fields[j] = fields[j], fields[i] - } - - matchFields := MatchFields(IgnoreExtras, - Fields{fields[0]: matcher}, - ) - - for i := 1; i < len(fields); i++ { - matchFields = MatchFields(IgnoreExtras, Fields{fields[i]: matchFields}) - } - - return matchFields -} - -func getKServeRouteName(isvc *kservev1beta1.InferenceService) string { - return isvc.Name + "-" + isvc.Namespace -} - -func waitForUpdatedGatewayCompletion(cli client.Client, op string, namespace, gatewayName string, isvcName string) (*istioclientv1beta1.Gateway, error) { - ctx := context.Background() - portName := fmt.Sprintf("%s-%s", "https", isvcName) - gateway := &istioclientv1beta1.Gateway{} - - // Get the Gateway resource - err := cli.Get(ctx, client.ObjectKey{Namespace: namespace, Name: gatewayName}, gateway) - if err != nil { - return nil, fmt.Errorf("failed to get Gateway: %w", err) - } - - // Check conditions based on operation (op) - switch op { - case "add": - if !hasServerFromGateway(gateway, portName) { - return nil, fmt.Errorf("server %s not found in Gateway %s", portName, gatewayName) - } - case "delete": - if hasServerFromGateway(gateway, portName) { - return nil, fmt.Errorf("server %s still exists in Gateway %s", portName, gatewayName) - } - default: - return nil, fmt.Errorf("unsupported operation: %s", op) - } - - return gateway, nil -} - -// checks if the server exists for the given gateway -func hasServerFromGateway(gateway *istioclientv1beta1.Gateway, portName string) bool { - targetServerExist := false - for _, server := range gateway.Spec.Servers { - if server.Port.Name == portName { - targetServerExist = true - break - } - } - return targetServerExist -} diff --git a/controllers/kserve_raw_inferenceservice_controller_test.go b/controllers/kserve_raw_inferenceservice_controller_test.go deleted file mode 100644 index e8d7cf9e..00000000 --- a/controllers/kserve_raw_inferenceservice_controller_test.go +++ /dev/null @@ -1,189 +0,0 @@ -package controllers - -import ( - "errors" - - kservev1alpha1 "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" - kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/opendatahub-io/odh-model-controller/controllers/constants" - routev1 "github.com/openshift/api/route/v1" - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - apierrs "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/intstr" -) - -var _ = Describe("The KServe Raw reconciler", func() { - var testNs string - createServingRuntime := func(namespace, path string) *kservev1alpha1.ServingRuntime { - servingRuntime := &kservev1alpha1.ServingRuntime{} - err := convertToStructuredResource(path, servingRuntime) - Expect(err).NotTo(HaveOccurred()) - servingRuntime.SetNamespace(namespace) - if err := cli.Create(ctx, servingRuntime); err != nil && !apierrs.IsAlreadyExists(err) { - Expect(err).NotTo(HaveOccurred()) - } - return servingRuntime - } - - createInferenceService := func(namespace, name string, path string) *kservev1beta1.InferenceService { - inferenceService := &kservev1beta1.InferenceService{} - err := convertToStructuredResource(path, inferenceService) - Expect(err).NotTo(HaveOccurred()) - inferenceService.SetNamespace(namespace) - if len(name) != 0 { - inferenceService.Name = name - } - inferenceService.Annotations = map[string]string{} - inferenceService.Annotations["serving.kserve.io/deploymentMode"] = "RawDeployment" - return inferenceService - } - - BeforeEach(func() { - testNs = Namespaces.Create(cli).Name - - inferenceServiceConfig := &corev1.ConfigMap{} - Expect(convertToStructuredResource(InferenceServiceConfigPath1, inferenceServiceConfig)).To(Succeed()) - if err := cli.Create(ctx, inferenceServiceConfig); err != nil && !apierrs.IsAlreadyExists(err) { - Fail(err.Error()) - } - - }) - - When("deploying a Kserve RawDeployment model", func() { - It("it should create a default clusterrolebinding for auth", func() { - _ = createServingRuntime(testNs, KserveServingRuntimePath1) - inferenceService := createInferenceService(testNs, KserveOvmsInferenceServiceName, KserveInferenceServicePath1) - if err := cli.Create(ctx, inferenceService); err != nil && !apierrs.IsAlreadyExists(err) { - Expect(err).NotTo(HaveOccurred()) - } - - crb := &rbacv1.ClusterRoleBinding{} - Eventually(func() error { - key := types.NamespacedName{Name: inferenceService.Namespace + "-" + constants.KserveServiceAccountName + "-auth-delegator", - Namespace: inferenceService.Namespace} - return cli.Get(ctx, key, crb) - }, timeout, interval).ShouldNot(HaveOccurred()) - - route := &routev1.Route{} - Eventually(func() error { - key := types.NamespacedName{Name: inferenceService.Name, Namespace: inferenceService.Namespace} - return cli.Get(ctx, key, route) - }, timeout, interval).Should(HaveOccurred()) - }) - It("it should create a custom rolebinding if isvc has a SA defined", func() { - serviceAccountName := "custom-sa" - _ = createServingRuntime(testNs, KserveServingRuntimePath1) - inferenceService := createInferenceService(testNs, KserveOvmsInferenceServiceName, KserveInferenceServicePath1) - inferenceService.Spec.Predictor.ServiceAccountName = serviceAccountName - if err := cli.Create(ctx, inferenceService); err != nil && !apierrs.IsAlreadyExists(err) { - Expect(err).NotTo(HaveOccurred()) - } - - crb := &rbacv1.ClusterRoleBinding{} - Eventually(func() error { - key := types.NamespacedName{Name: inferenceService.Namespace + "-" + serviceAccountName + "-auth-delegator", - Namespace: inferenceService.Namespace} - return cli.Get(ctx, key, crb) - }, timeout, interval).ShouldNot(HaveOccurred()) - }) - It("it should create a route if isvc has the label to expose route", func() { - inferenceService := createInferenceService(testNs, KserveOvmsInferenceServiceName, KserveInferenceServicePath1) - inferenceService.Labels = map[string]string{} - inferenceService.Labels[constants.KserveNetworkVisibility] = constants.LabelEnableKserveRawRoute - // The service is manually created before the isvc otherwise the unit test risks running into a race condition - // where the reconcile loop finishes before the service is created, leading to no route being created. - isvcService := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: KserveOvmsInferenceServiceName + "-predictor", - Namespace: inferenceService.Namespace, - Annotations: map[string]string{ - "openshift.io/display-name": KserveOvmsInferenceServiceName, - "serving.kserve.io/deploymentMode": "RawDeployment", - }, - Labels: map[string]string{ - "app": "isvc." + KserveOvmsInferenceServiceName + "-predictor", - "component": "predictor", - "serving.kserve.io/inferenceservice": KserveOvmsInferenceServiceName, - }, - }, - Spec: corev1.ServiceSpec{ - ClusterIP: "None", - IPFamilies: []corev1.IPFamily{"IPv4"}, - Ports: []corev1.ServicePort{ - { - Name: "https", - Protocol: corev1.ProtocolTCP, - Port: 8888, - TargetPort: intstr.FromString("https"), - }, - }, - ClusterIPs: []string{"None"}, - Selector: map[string]string{ - "app": "isvc." + KserveOvmsInferenceServiceName + "-predictor", - }, - }, - } - if err := cli.Create(ctx, isvcService); err != nil && !apierrs.IsAlreadyExists(err) { - Expect(err).NotTo(HaveOccurred()) - } - service := &corev1.Service{} - Eventually(func() error { - key := types.NamespacedName{Name: isvcService.Name, Namespace: isvcService.Namespace} - return cli.Get(ctx, key, service) - }, timeout, interval).Should(Succeed()) - - _ = createServingRuntime(testNs, KserveServingRuntimePath1) - if err := cli.Create(ctx, inferenceService); err != nil && !apierrs.IsAlreadyExists(err) { - Expect(err).NotTo(HaveOccurred()) - } - - route := &routev1.Route{} - Eventually(func() error { - key := types.NamespacedName{Name: inferenceService.Name, Namespace: inferenceService.Namespace} - return cli.Get(ctx, key, route) - }, timeout, interval).ShouldNot(HaveOccurred()) - }) - }) - When("deleting a Kserve RawDeployment model", func() { - It("the associated route should be deleted", func() { - _ = createServingRuntime(testNs, KserveServingRuntimePath1) - inferenceService := createInferenceService(testNs, KserveOvmsInferenceServiceName, KserveInferenceServicePath1) - if err := cli.Create(ctx, inferenceService); err != nil && !apierrs.IsAlreadyExists(err) { - Expect(err).NotTo(HaveOccurred()) - } - - Expect(cli.Delete(ctx, inferenceService)).Should(Succeed()) - - route := &routev1.Route{} - Eventually(func() error { - key := types.NamespacedName{Name: inferenceService.Name, Namespace: inferenceService.Namespace} - return cli.Get(ctx, key, route) - }, timeout, interval).Should(HaveOccurred()) - }) - }) - When("namespace no longer has any RawDeployment models", func() { - It("should delete the default clusterrolebinding", func() { - _ = createServingRuntime(testNs, KserveServingRuntimePath1) - inferenceService := createInferenceService(testNs, KserveOvmsInferenceServiceName, KserveInferenceServicePath1) - if err := cli.Create(ctx, inferenceService); err != nil && !apierrs.IsAlreadyExists(err) { - Expect(err).NotTo(HaveOccurred()) - } - Expect(cli.Delete(ctx, inferenceService)).Should(Succeed()) - crb := &rbacv1.ClusterRoleBinding{} - Eventually(func() error { - namespacedNamed := types.NamespacedName{Name: testNs + "-" + constants.KserveServiceAccountName + "-auth-delegator", Namespace: WorkingNamespace} - err := cli.Get(ctx, namespacedNamed, crb) - if apierrs.IsNotFound(err) { - return nil - } else { - return errors.New("crb deletion not detected") - } - }, timeout, interval).ShouldNot(HaveOccurred()) - }) - }) -}) diff --git a/controllers/mm_inferenceservice_controller_test.go b/controllers/mm_inferenceservice_controller_test.go deleted file mode 100644 index 75d0cc55..00000000 --- a/controllers/mm_inferenceservice_controller_test.go +++ /dev/null @@ -1,88 +0,0 @@ -/* - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -import ( - "context" - kservev1alpha1 "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" - kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - "github.com/opendatahub-io/odh-model-controller/controllers/comparators" - routev1 "github.com/openshift/api/route/v1" - "k8s.io/apimachinery/pkg/types" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -var _ = Describe("The Openshift model controller", func() { - - When("creating a ServiceRuntime & InferenceService with 'enable-route' enabled", func() { - - BeforeEach(func() { - ctx := context.Background() - - servingRuntime1 := &kservev1alpha1.ServingRuntime{} - err := convertToStructuredResource(ServingRuntimePath1, servingRuntime1) - Expect(err).NotTo(HaveOccurred()) - Expect(cli.Create(ctx, servingRuntime1)).Should(Succeed()) - - servingRuntime2 := &kservev1alpha1.ServingRuntime{} - err = convertToStructuredResource(ServingRuntimePath2, servingRuntime2) - Expect(err).NotTo(HaveOccurred()) - Expect(cli.Create(ctx, servingRuntime2)).Should(Succeed()) - }) - - It("when InferenceService specifies a runtime, should create a Route to expose the traffic externally", func() { - inferenceService := &kservev1beta1.InferenceService{} - err := convertToStructuredResource(InferenceService1, inferenceService) - Expect(err).NotTo(HaveOccurred()) - Expect(cli.Create(ctx, inferenceService)).Should(Succeed()) - - By("By checking that the controller has created the Route") - - route := &routev1.Route{} - Eventually(func() error { - key := types.NamespacedName{Name: inferenceService.Name, Namespace: inferenceService.Namespace} - return cli.Get(ctx, key, route) - }, timeout, interval).ShouldNot(HaveOccurred()) - - expectedRoute := &routev1.Route{} - err = convertToStructuredResource(ExpectedRoutePath, expectedRoute) - Expect(err).NotTo(HaveOccurred()) - - Expect(comparators.GetMMRouteComparator()(route, expectedRoute)).Should(BeTrue()) - }) - - It("when InferenceService does not specifies a runtime, should automatically pick a runtime and create a Route", func() { - inferenceService := &kservev1beta1.InferenceService{} - err := convertToStructuredResource(InferenceServiceNoRuntime, inferenceService) - Expect(err).NotTo(HaveOccurred()) - Expect(cli.Create(ctx, inferenceService)).Should(Succeed()) - - route := &routev1.Route{} - Eventually(func() error { - key := types.NamespacedName{Name: inferenceService.Name, Namespace: inferenceService.Namespace} - return cli.Get(ctx, key, route) - }, timeout, interval).ShouldNot(HaveOccurred()) - - expectedRoute := &routev1.Route{} - err = convertToStructuredResource(ExpectedRouteNoRuntimePath, expectedRoute) - Expect(err).NotTo(HaveOccurred()) - - Expect(comparators.GetMMRouteComparator()(route, expectedRoute)).Should(BeTrue()) - }) - }) -}) diff --git a/controllers/suite_test.go b/controllers/suite_test.go deleted file mode 100644 index ff1f3448..00000000 --- a/controllers/suite_test.go +++ /dev/null @@ -1,333 +0,0 @@ -/* - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -import ( - "context" - "fmt" - "math/rand" - "os" - "path/filepath" - "testing" - "time" - - apierrs "k8s.io/apimachinery/pkg/api/errors" - - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "sigs.k8s.io/controller-runtime/pkg/metrics/server" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - kservev1alpha1 "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" - kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - authorinov1beta2 "github.com/kuadrant/authorino/api/v1beta2" - monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" - "go.uber.org/zap/zapcore" - k8srbacv1 "k8s.io/api/rbac/v1" - "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - routev1 "github.com/openshift/api/route/v1" - istioclientv1beta1 "istio.io/client-go/pkg/apis/networking/v1beta1" - corev1 "k8s.io/api/core/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/kubernetes/fake" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - - "github.com/opendatahub-io/odh-model-controller/controllers/utils" - //+kubebuilder:scaffold:imports -) - -// These tests use Ginkgo (BDD-style Go testing framework). Refer to -// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. - -// +kubebuilder:docs-gen:collapse=Imports - -var ( - cli client.Client - envTest *envtest.Environment - ctx context.Context - cancel context.CancelFunc - Namespaces NamespaceHolder -) - -const ( - WorkingNamespace = "default" - MonitoringNS = "monitoring-ns" - RoleBindingPath = "./testdata/results/model-server-ns-role.yaml" - ServingRuntimePath1 = "./testdata/deploy/test-openvino-serving-runtime-1.yaml" - KserveServingRuntimePath1 = "./testdata/deploy/kserve-openvino-serving-runtime-1.yaml" - ServingRuntimePath2 = "./testdata/deploy/test-openvino-serving-runtime-2.yaml" - InferenceService1 = "./testdata/deploy/openvino-inference-service-1.yaml" - InferenceServiceNoRuntime = "./testdata/deploy/openvino-inference-service-no-runtime.yaml" - KserveInferenceServicePath1 = "./testdata/deploy/kserve-openvino-inference-service-1.yaml" - InferenceServiceConfigPath1 = "./testdata/configmaps/inferenceservice-config.yaml" - ExpectedRoutePath = "./testdata/results/example-onnx-mnist-route.yaml" - ExpectedRouteNoRuntimePath = "./testdata/results/example-onnx-mnist-no-runtime-route.yaml" - DSCIWithAuthorization = "./testdata/dsci-with-authorino-enabled.yaml" - DSCIWithoutAuthorization = "./testdata/dsci-with-authorino-missing.yaml" - KServeAuthorizationPolicy = "./testdata/kserve-authorization-policy.yaml" - odhtrustedcabundleConfigMapPath = "./testdata/configmaps/odh-trusted-ca-bundle-configmap.yaml" - odhKserveCustomCABundleConfigMapPath = "./testdata/configmaps/odh-kserve-custom-ca-cert-configmap.yaml" - timeout = time.Second * 20 - interval = time.Millisecond * 10 -) - -func init() { - rand.Seed(time.Now().UnixNano()) - Namespaces = NamespaceHolder{} -} - -func TestAPIs(t *testing.T) { - RegisterFailHandler(Fail) - - RunSpecs(t, "Controller & Webhook Suite") -} - -var _ = BeforeSuite(func() { - ctx, cancel = context.WithCancel(context.TODO()) - - // Initialize logger - opts := zap.Options{ - Development: true, - TimeEncoder: zapcore.TimeEncoderOfLayout(time.RFC3339), - } - logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseFlagOptions(&opts))) - - // Initialize test environment: - By("Bootstrapping test environment") - envTest = &envtest.Environment{ - CRDInstallOptions: envtest.CRDInstallOptions{ - Paths: []string{ - filepath.Join("..", "config", "crd", "bases"), - filepath.Join("..", "config", "crd", "external"), - }, - ErrorIfPathMissing: true, - CleanUpAfterUse: false, - }, - } - - cfg, err := envTest.Start() - Expect(err).NotTo(HaveOccurred()) - Expect(cfg).NotTo(BeNil()) - - // Register API objects - testScheme := runtime.NewScheme() - utils.RegisterSchemes(testScheme) - utilruntime.Must(authorinov1beta2.AddToScheme(testScheme)) - utilruntime.Must(istioclientv1beta1.AddToScheme(testScheme)) - - // +kubebuilder:scaffold:scheme - - // Initialize Kubernetes client - cli, err = client.New(cfg, client.Options{Scheme: testScheme}) - Expect(err).NotTo(HaveOccurred()) - Expect(cli).NotTo(BeNil()) - - // Create istio-system namespace - _, meshNamespace := utils.GetIstioControlPlaneName(ctx, cli) - istioNamespace := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: meshNamespace, - Namespace: meshNamespace, - }, - } - Expect(cli.Create(ctx, istioNamespace)).Should(Succeed()) - - // Setup controller manager - mgr, err := ctrl.NewManager(cfg, ctrl.Options{ - Scheme: testScheme, - LeaderElection: false, - Metrics: server.Options{ - BindAddress: "0", - }, - }) - - Expect(err).NotTo(HaveOccurred()) - - err = (NewOpenshiftInferenceServiceReconciler( - mgr.GetClient(), - mgr.GetAPIReader(), - ctrl.Log.WithName("controllers").WithName("InferenceService-controller"), - false)). - SetupWithManager(mgr) - Expect(err).ToNot(HaveOccurred()) - - err = (&MonitoringReconciler{ - Client: cli, - Log: ctrl.Log.WithName("controllers").WithName("monitoring-controller"), - MonitoringNS: MonitoringNS, - }).SetupWithManager(mgr) - Expect(err).ToNot(HaveOccurred()) - - err = (&StorageSecretReconciler{ - Client: cli, - Log: ctrl.Log.WithName("controllers").WithName("Storage-Secret-Controller"), - }).SetupWithManager(mgr) - Expect(err).ToNot(HaveOccurred()) - - err = (NewModelRegistryInferenceServiceReconciler( - cli, - ctrl.Log.WithName("controllers").WithName("ModelRegistry-InferenceService-Controller"), - false, - "", - )).SetupWithManager(mgr) - Expect(err).ToNot(HaveOccurred()) - - err = (&KServeCustomCACertReconciler{ - Client: cli, - Log: ctrl.Log.WithName("controllers").WithName("KServe-Custom-CA-Bundle-ConfigMap-Controller"), - }).SetupWithManager(mgr) - Expect(err).ToNot(HaveOccurred()) - - kclient, _ := kubernetes.NewForConfig(cfg) - err = (&NimAccountReconciler{ - Client: cli, - Log: ctrl.Log.WithName("controllers").WithName("NimAccountReconciler"), - KClient: kclient, - }).SetupWithManager(mgr, ctx) - Expect(err).ToNot(HaveOccurred()) - - // Start the manager - go func() { - defer GinkgoRecover() - err = mgr.Start(ctx) - Expect(err).ToNot(HaveOccurred(), "Failed to run manager") - }() - -}, 60) - -var _ = AfterSuite(func() { - cancel() - By("Tearing down the test environment") - err := envTest.Stop() - Expect(err).NotTo(HaveOccurred()) -}) - -// Cleanup resources to not contaminate between tests -var _ = AfterEach(func() { - cleanUp := func(namespace string, cli client.Client) { - inNamespace := client.InNamespace(namespace) - _, meshNamespace := utils.GetIstioControlPlaneName(ctx, cli) - istioNamespace := client.InNamespace(meshNamespace) - Expect(cli.DeleteAllOf(context.TODO(), &kservev1alpha1.ServingRuntime{}, inNamespace)).ToNot(HaveOccurred()) - Expect(cli.DeleteAllOf(context.TODO(), &kservev1beta1.InferenceService{}, inNamespace)).ToNot(HaveOccurred()) - Expect(cli.DeleteAllOf(context.TODO(), &routev1.Route{}, inNamespace)).ToNot(HaveOccurred()) - Expect(cli.DeleteAllOf(context.TODO(), &monitoringv1.ServiceMonitor{}, inNamespace)).ToNot(HaveOccurred()) - Expect(cli.DeleteAllOf(context.TODO(), &k8srbacv1.RoleBinding{}, inNamespace)).ToNot(HaveOccurred()) - Expect(cli.DeleteAllOf(context.TODO(), &corev1.Secret{}, inNamespace)).ToNot(HaveOccurred()) - Expect(cli.DeleteAllOf(context.TODO(), &authorinov1beta2.AuthConfig{}, inNamespace)).ToNot(HaveOccurred()) - Expect(cli.DeleteAllOf(context.TODO(), &corev1.ConfigMap{}, inNamespace)).ToNot(HaveOccurred()) - Expect(cli.DeleteAllOf(context.TODO(), &corev1.Service{}, inNamespace)).ToNot(HaveOccurred()) - Expect(cli.DeleteAllOf(context.TODO(), &istioclientv1beta1.Gateway{}, istioNamespace)).ToNot(HaveOccurred()) - } - cleanUp(WorkingNamespace, cli) - for _, ns := range Namespaces.All() { - cleanUp(ns, cli) - } - Namespaces.Clear() -}) - -func convertToStructuredResource(path string, out runtime.Object) error { - data, err := os.ReadFile(path) - if err != nil { - return err - } - - return utils.ConvertToStructuredResource(data, out) -} - -func convertToUnstructuredResource(path string, out *unstructured.Unstructured) error { - data, err := os.ReadFile(path) - if err != nil { - return err - } - - return utils.ConvertToUnstructuredResource(data, out) -} - -type NamespaceHolder struct { - namespaces []string -} - -func (n *NamespaceHolder) Get() string { - ns := createTestNamespaceName() - n.namespaces = append(n.namespaces, ns) - return ns -} - -func (n *NamespaceHolder) All() []string { - return n.namespaces -} - -func (n *NamespaceHolder) Clear() { - n.namespaces = []string{} -} - -func (n *NamespaceHolder) Create(cli client.Client) *corev1.Namespace { - testNs := Namespaces.Get() - testNamespace := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: testNs, - Namespace: testNs, - }, - } - Expect(cli.Create(ctx, testNamespace)).Should(Succeed()) - return testNamespace -} - -func createTestNamespaceName() string { - n := 5 - letterRunes := []rune("abcdefghijklmnopqrstuvwxyz") - - b := make([]rune, n) - for i := range b { - b[i] = letterRunes[rand.Intn(len(letterRunes))] - } - return "test-ns-" + string(b) -} - -func NewFakeClientsetWrapper(fakeClient *fake.Clientset) kubernetes.Interface { - return fakeClient -} - -func waitForConfigMap(cli client.Client, namespace, configMapName string, maxTries int, delay time.Duration) (*corev1.ConfigMap, error) { - time.Sleep(delay) - - ctx := context.Background() - configMap := &corev1.ConfigMap{} - for try := 1; try <= maxTries; try++ { - err := cli.Get(ctx, client.ObjectKey{Namespace: namespace, Name: configMapName}, configMap) - if err == nil { - return configMap, nil - } - if !apierrs.IsNotFound(err) { - return nil, fmt.Errorf("failed to get configmap %s/%s: %v", namespace, configMapName, err) - } - - if try > maxTries { - time.Sleep(1 * time.Second) - return nil, err - } - } - return configMap, nil -} diff --git a/controllers/webhook/isvc_validator.go b/controllers/webhook/isvc_validator.go deleted file mode 100644 index 0fa43ef6..00000000 --- a/controllers/webhook/isvc_validator.go +++ /dev/null @@ -1,84 +0,0 @@ -/* - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package webhook - -import ( - "context" - "fmt" - kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - "github.com/opendatahub-io/odh-model-controller/controllers/utils" - "net/http" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - - logf "sigs.k8s.io/controller-runtime/pkg/log" -) - -// +kubebuilder:webhook:admissionReviewVersions=v1beta1,path=/validate-isvc-odh-service,mutating=false,failurePolicy=fail,groups="serving.kserve.io",resources=inferenceservices,verbs=create,versions=v1beta1,name=validating.isvc.odh-model-controller.opendatahub.io,sideEffects=None - -type IsvcValidator struct { - Client client.Client - Decoder *admission.Decoder -} - -// NewIsvcValidator For tests purposes -func NewIsvcValidator(client client.Client, decoder *admission.Decoder) *IsvcValidator { - return &IsvcValidator{ - Client: client, - Decoder: decoder, - } -} - -// Handle implements admission.Validator so a webhook will be registered for the type serving.kserve.io.inferenceServices -// This webhook will filter out the protected namespaces preventing the user to create the InferenceService in those namespaces -// which are: The Istio control plane namespace, Application namespace and Serving namespace -// func (is *isvcValidator) ValidateCreate(ctx context.Context, obj runtime.Object) (warnings admission.Warnings, err error) { -func (is *IsvcValidator) Handle(ctx context.Context, req admission.Request) admission.Response { - protectedNamespaces := make([]string, 3) - // hardcoding for now since there is no plan to install knative on other namespaces - protectedNamespaces[0] = "knative-serving" - - log := logf.FromContext(ctx).WithName("InferenceServiceValidatingWebhook") - - isvc := &kservev1beta1.InferenceService{} - errs := is.Decoder.Decode(req, isvc) - if errs != nil { - return admission.Errored(http.StatusBadRequest, errs) - } - - log = log.WithValues("namespace", isvc.Namespace, "isvc", isvc.Name) - log.Info("Validating InferenceService") - - _, meshNamespace := utils.GetIstioControlPlaneName(ctx, is.Client) - protectedNamespaces[1] = meshNamespace - - appNamespace, err := utils.GetApplicationNamespace(ctx, is.Client) - if err != nil { - return admission.Errored(http.StatusBadRequest, err) - } - protectedNamespaces[2] = appNamespace - - log.Info("Filtering protected namespaces", "namespaces", protectedNamespaces) - for _, ns := range protectedNamespaces { - if isvc.Namespace == ns { - log.V(1).Info("Namespace is protected, the InferenceService will not be created") - return admission.Denied(fmt.Sprintf("The InferenceService %s "+ - "cannot be created in protected namespace %s.", isvc.Name, isvc.Namespace)) - } - } - log.Info("Namespace is not protected") - return admission.Allowed("Namespace is not protected") -} diff --git a/controllers/webhook/ksvc_validator.go b/controllers/webhook/ksvc_validator.go deleted file mode 100644 index 4769c3e4..00000000 --- a/controllers/webhook/ksvc_validator.go +++ /dev/null @@ -1,128 +0,0 @@ -/* - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package webhook - -import ( - "context" - "fmt" - "strings" - - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - - kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - "k8s.io/apimachinery/pkg/runtime" - types2 "k8s.io/apimachinery/pkg/types" - knservingv1 "knative.dev/serving/pkg/apis/serving/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" - - "github.com/opendatahub-io/odh-model-controller/controllers/constants" - "github.com/opendatahub-io/odh-model-controller/controllers/resources" - "github.com/opendatahub-io/odh-model-controller/controllers/utils" -) - -// +kubebuilder:webhook:admissionReviewVersions=v1,path=/validate-serving-knative-dev-v1-service,mutating=false,failurePolicy=fail,groups="serving.knative.dev",resources=services,verbs=create,versions=v1,name=validating.ksvc.odh-model-controller.opendatahub.io,sideEffects=None - -type ksvcValidator struct { - client client.Client -} - -func NewKsvcValidator(client client.Client) *ksvcValidator { - return &ksvcValidator{client: client} -} - -func (v *ksvcValidator) ValidateCreate(ctx context.Context, obj runtime.Object) (warnings admission.Warnings, err error) { - log := logf.FromContext(ctx).WithName("KsvcValidatingWebhook") - ksvc, ok := obj.(*knservingv1.Service) - if !ok { - return nil, fmt.Errorf("expected a Knative Service but got a %T", obj) - } - - log = log.WithValues("namespace", ksvc.Namespace, "ksvc", ksvc.Name) - log.Info("Validating Knative Service") - - // If there is an explicit intent for not having a sidecar, skip validation - ksvcTemplateMeta := ksvc.Spec.Template.GetObjectMeta() - if ksvcTemplateMeta != nil { - if templateAnnotations := ksvcTemplateMeta.GetAnnotations(); templateAnnotations[constants.IstioSidecarInjectAnnotationName] == "false" { - log.V(1).Info("Skipping validation of Knative Service because there is an explicit intent to exclude it from the Service Mesh") - return nil, nil - } - } - - // Only validate the KSVC if it is owned by KServe controller - ksvcMetadata := ksvc.GetObjectMeta() - if ksvcMetadata == nil { - log.V(1).Info("Skipping validation of Knative Service because it does not have metadata") - return nil, nil - } - ksvcOwnerReferences := ksvcMetadata.GetOwnerReferences() - if ksvcOwnerReferences == nil { - log.V(1).Info("Skipping validation of Knative Service because it does not have owner references") - return nil, nil - } - isOwnedByKServe := false - for _, owner := range ksvcOwnerReferences { - if owner.Kind == constants.InferenceServiceKind { - if strings.Contains(owner.APIVersion, kservev1beta1.SchemeGroupVersion.Group) { - isOwnedByKServe = true - } - } - } - if !isOwnedByKServe { - log.V(1).Info("Skipping validation of Knative Service because it is not owned by KServe") - return nil, nil - } - - // Since the Ksvc is owned by an InferenceService, it is known that it is required to be - // in the Mesh. Thus, the involved namespace needs to be enrolled in the mesh. - // Go and check the ServiceMeshMemberRoll to verify that the namespace is already a - // member. If it is still not a member, reject creation of the Ksvc to prevent - // creation of a Pod that would not be in the Mesh. - smmrQuerier := resources.NewServiceMeshMemberRole(v.client) - _, meshNamespace := utils.GetIstioControlPlaneName(ctx, v.client) - smmr, fetchSmmrErr := smmrQuerier.FetchSMMR(ctx, log, types2.NamespacedName{Name: constants.ServiceMeshMemberRollName, Namespace: meshNamespace}) - if fetchSmmrErr != nil { - log.Error(fetchSmmrErr, "Error when fetching ServiceMeshMemberRoll", "smmr.namespace", meshNamespace, "smmr.name", constants.ServiceMeshMemberRollName) - return nil, fetchSmmrErr - } - if smmr == nil { - log.Info("Rejecting Knative service because the ServiceMeshMemberRoll does not exist", "smmr.namespace", meshNamespace, "smmr.name", constants.ServiceMeshMemberRollName) - return nil, fmt.Errorf("rejecting creation of Knative service %s on namespace %s because the ServiceMeshMemberRoll does not exist", ksvc.Name, ksvc.Namespace) - } - - log = log.WithValues("smmr.namespace", smmr.Namespace, "smmr.name", smmr.Name) - - for _, memberNamespace := range smmr.Status.ConfiguredMembers { - if memberNamespace == ksvc.Namespace { - log.V(1).Info("The Knative service is accepted") - return nil, nil - } - } - - log.Info("Rejecting Knative service because its namespace is not a member of the service mesh") - return nil, fmt.Errorf("rejecting creation of Knative service %s on namespace %s because the namespace is not a configured member of the mesh", ksvc.Name, ksvc.Namespace) -} - -func (v *ksvcValidator) ValidateUpdate(_ context.Context, _, _ runtime.Object) (warnings admission.Warnings, err error) { - // Nothing to validate on updates - return nil, nil -} - -func (v *ksvcValidator) ValidateDelete(_ context.Context, _ runtime.Object) (warnings admission.Warnings, err error) { - // For deletion, we don't need to validate anything. - return nil, nil -} diff --git a/controllers/webhook/nim_account_webhook.go b/controllers/webhook/nim_account_webhook.go deleted file mode 100644 index 97bdd6f2..00000000 --- a/controllers/webhook/nim_account_webhook.go +++ /dev/null @@ -1,79 +0,0 @@ -/* - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package webhook - -import ( - "context" - "fmt" - - nimv1 "github.com/opendatahub-io/odh-model-controller/api/nim/v1" - - "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" -) - -// +kubebuilder:webhook:admissionReviewVersions=v1,path=/validate-nim-opendatahub-io-v1-account,mutating=false,failurePolicy=fail,groups=nim.opendatahub.io,resources=accounts,verbs=create;update,versions=v1,name=validating.nim.account.odh-model-controller.opendatahub.io,sideEffects=None - -type nimAccountValidator struct { - client client.Client -} - -func NewNimAccountValidator(client client.Client) admission.CustomValidator { - return &nimAccountValidator{client: client} -} - -func (v *nimAccountValidator) ValidateCreate(ctx context.Context, obj runtime.Object) (warnings admission.Warnings, err error) { - account := obj.(*nimv1.Account) - - log := logf.FromContext(ctx).WithName("NIMAccountValidatingWebhook"). - WithValues("namespace", account.Namespace, "account", account.Name) - log.Info("Validating NIM Account creation") - - err = v.verifySingletonInNamespace(ctx, account) - if err != nil { - log.Error(err, "Rejecting NIM Account creation because checking singleton didn't pass") - return nil, err - } - - return nil, nil -} - -func (v *nimAccountValidator) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (warnings admission.Warnings, err error) { - // For update, nothing needs to be validated - return nil, nil -} - -func (v *nimAccountValidator) ValidateDelete(ctx context.Context, obj runtime.Object) (warnings admission.Warnings, err error) { - // For deletion, nothing needs to be validated - return nil, nil -} - -func (v *nimAccountValidator) verifySingletonInNamespace(ctx context.Context, account *nimv1.Account) error { - accountList := nimv1.AccountList{} - err := v.client.List(ctx, - &accountList, - client.InNamespace(account.Namespace)) - if err != nil { - return fmt.Errorf("failed to verify if there are existing Accounts with err: %s", err.Error()) - } - - if len(accountList.Items) > 0 { - return fmt.Errorf("rejecting creation of Account %s in namespace %s because there is already an Account created in the namespace", account.Name, account.Namespace) - } - return nil -} diff --git a/controllers/webhook_isvc_validator_test.go b/controllers/webhook_isvc_validator_test.go deleted file mode 100644 index c23320c5..00000000 --- a/controllers/webhook_isvc_validator_test.go +++ /dev/null @@ -1,108 +0,0 @@ -/* - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -import ( - "encoding/json" - kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/opendatahub-io/odh-model-controller/controllers/utils" - "github.com/opendatahub-io/odh-model-controller/controllers/webhook" - admissionv1 "k8s.io/api/admission/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" -) - -var _ = Describe("Inference Service validator webhook", func() { - var validator *webhook.IsvcValidator - var meshNamespace, appsNamespace string - - createInferenceService := func(namespace, name string) *kservev1beta1.InferenceService { - inferenceService := &kservev1beta1.InferenceService{} - err := convertToStructuredResource(KserveInferenceServicePath1, inferenceService) - Expect(err).NotTo(HaveOccurred()) - inferenceService.SetNamespace(namespace) - if len(name) != 0 { - inferenceService.Name = name - } - Expect(cli.Create(ctx, inferenceService)).Should(Succeed()) - return inferenceService - } - - BeforeEach(func() { - _, meshNamespace = utils.GetIstioControlPlaneName(ctx, cli) - appsNamespace, _ = utils.GetApplicationNamespace(ctx, cli) - validator = webhook.NewIsvcValidator(cli, admission.NewDecoder(cli.Scheme())) - }) - - It("Should allow the Inference Service in the test-model namespace", func() { - testNs := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-model", - Namespace: "test-model", - }, - } - Expect(cli.Create(ctx, testNs)).Should(Succeed()) - - isvc := createInferenceService(testNs.Name, "test-isvc") - isvcBytes, err := json.Marshal(isvc) - Expect(err).NotTo(HaveOccurred()) - response := validator.Handle(ctx, admission.Request{ - AdmissionRequest: admissionv1.AdmissionRequest{Namespace: testNs.Name, Name: "test-isvc", - Object: runtime.RawExtension{Raw: isvcBytes}}}) - Expect(response.Allowed).To(BeTrue()) - }) - - It("Should not allow the Inference Service in the ServiceMesh namespace", func() { - isvc := createInferenceService(meshNamespace, "test-isvc") - isvcBytes, err := json.Marshal(isvc) - Expect(err).NotTo(HaveOccurred()) - response := validator.Handle(ctx, admission.Request{ - AdmissionRequest: admissionv1.AdmissionRequest{Namespace: meshNamespace, Name: "test-isvc", - Object: runtime.RawExtension{Raw: isvcBytes}}}) - Expect(response.Allowed).To(BeFalse()) - }) - - It("Should not allow the Inference Service in the ApplicationsNamespace namespace", func() { - isvc := createInferenceService(appsNamespace, "test-isvc") - isvcBytes, err := json.Marshal(isvc) - Expect(err).NotTo(HaveOccurred()) - response := validator.Handle(ctx, admission.Request{ - AdmissionRequest: admissionv1.AdmissionRequest{Namespace: appsNamespace, Name: "test-isvc", - Object: runtime.RawExtension{Raw: isvcBytes}}}) - Expect(response.Allowed).To(BeFalse()) - }) - - It("Should not allow the Inference Service in the knative-serving namespace", func() { - testNs := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "knative-serving", - Namespace: "knative-serving", - }, - } - Expect(cli.Create(ctx, testNs)).Should(Succeed()) - isvc := createInferenceService(testNs.Name, "test-isvc") - isvcBytes, err := json.Marshal(isvc) - Expect(err).NotTo(HaveOccurred()) - response := validator.Handle(ctx, admission.Request{ - AdmissionRequest: admissionv1.AdmissionRequest{Namespace: testNs.Name, Name: "test-isvc", - Object: runtime.RawExtension{Raw: isvcBytes}}}) - Expect(response.Allowed).To(BeFalse()) - }) -}) diff --git a/go.mod b/go.mod index b97e8935..dda40b92 100644 --- a/go.mod +++ b/go.mod @@ -1,135 +1,146 @@ module github.com/opendatahub-io/odh-model-controller -go 1.21 +go 1.22.7 + +toolchain go1.22.9 require ( - github.com/go-logr/logr v1.4.1 + github.com/go-logr/logr v1.4.2 github.com/hashicorp/errwrap v1.1.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/kserve/kserve v0.12.1 - github.com/kuadrant/authorino v0.15.0 + github.com/kserve/kserve v0.14.0 + github.com/kuadrant/authorino v0.18.1 github.com/kubeflow/model-registry v0.2.11 github.com/onsi/ginkgo v1.16.5 - github.com/onsi/gomega v1.30.0 - github.com/openshift/api v3.9.0+incompatible + github.com/onsi/ginkgo/v2 v2.20.1 + github.com/onsi/gomega v1.34.2 + github.com/openshift/api v0.0.0-20240912201240-0a8800162826 github.com/pkg/errors v0.9.1 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.64.1 - github.com/tidwall/gjson v1.17.0 - go.uber.org/zap v1.26.0 - gopkg.in/yaml.v2 v2.4.0 - istio.io/api v1.21.5 - istio.io/client-go v1.21.5 - k8s.io/api v0.29.0 - k8s.io/apimachinery v0.29.0 - k8s.io/client-go v0.29.0 - k8s.io/utils v0.0.0-20230726121419-3b25d923346b - knative.dev/pkg v0.0.0-20231115001034-97c7258e3a98 - knative.dev/serving v0.39.3 - maistra.io/api v0.0.0-20230417135504-0536f6c22b1c - sigs.k8s.io/controller-runtime v0.16.3 + github.com/tidwall/gjson v1.17.3 + istio.io/api v1.23.0 + istio.io/client-go v1.23.0 + k8s.io/api v0.31.0 + k8s.io/apimachinery v0.31.0 + k8s.io/client-go v0.31.0 + k8s.io/utils v0.0.0-20240821151609-f90d01438635 + knative.dev/pkg v0.0.0-20240815051656-89743d9bbf7c + knative.dev/serving v0.42.2 + maistra.io/api v0.0.0-20240319144440-ffa91c765143 + sigs.k8s.io/controller-runtime v0.19.1 sigs.k8s.io/yaml v1.4.0 ) require ( - cloud.google.com/go v0.112.1 // indirect + cloud.google.com/go v0.115.1 // indirect + cloud.google.com/go/auth v0.9.1 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect cloud.google.com/go/compute/metadata v0.5.0 // indirect - cloud.google.com/go/iam v1.1.6 // indirect - cloud.google.com/go/storage v1.38.0 // indirect - github.com/aws/aws-sdk-go v1.48.0 // indirect + cloud.google.com/go/iam v1.2.0 // indirect + cloud.google.com/go/storage v1.43.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect + github.com/aws/aws-sdk-go v1.55.5 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect github.com/blendle/zapdriver v1.3.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/evanphx/json-patch v5.7.0+incompatible // indirect - github.com/evanphx/json-patch/v5 v5.7.0 // indirect + github.com/emicklei/go-restful/v3 v3.12.1 // indirect + github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect - github.com/go-openapi/jsonpointer v0.20.0 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.22.4 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/cel-go v0.20.1 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/google/go-containerregistry v0.16.1 // indirect + github.com/google/go-containerregistry v0.20.2 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/s2a-go v0.1.7 // indirect + github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect + github.com/google/s2a-go v0.1.8 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect - github.com/googleapis/gax-go/v2 v2.12.3 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.3 // indirect + github.com/googleapis/gax-go/v2 v2.13.0 // indirect github.com/googleapis/google-cloud-go-testing v0.0.0-20210719221736-1c9a4c676720 // indirect - github.com/imdario/mergo v0.3.16 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect + github.com/imdario/mergo v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nxadm/tail v1.4.8 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/prometheus/client_golang v1.17.0 // indirect - github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/common v0.45.0 // indirect - github.com/prometheus/procfs v0.12.0 // indirect + github.com/prometheus/client_golang v1.20.2 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.57.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stoewer/go-strcase v1.2.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect + github.com/x448/float16 v0.8.4 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect - go.opentelemetry.io/otel v1.24.0 // indirect - go.opentelemetry.io/otel/metric v1.24.0 // indirect - go.opentelemetry.io/otel/trace v1.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect + go.opentelemetry.io/otel v1.29.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 // indirect + go.opentelemetry.io/otel/metric v1.29.0 // indirect + go.opentelemetry.io/otel/sdk v1.28.0 // indirect + go.opentelemetry.io/otel/trace v1.29.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.26.0 // indirect - golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect + golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 // indirect golang.org/x/net v0.28.0 // indirect golang.org/x/oauth2 v0.22.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.24.0 // indirect golang.org/x/term v0.23.0 // indirect golang.org/x/text v0.17.0 // indirect - golang.org/x/time v0.5.0 // indirect + golang.org/x/time v0.6.0 // indirect + golang.org/x/tools v0.24.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/api v0.171.0 // indirect - google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect + google.golang.org/api v0.195.0 // indirect + google.golang.org/genproto v0.0.0-20240827150818-7e3bb234dfed // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed // indirect google.golang.org/grpc v1.67.1 // indirect google.golang.org/protobuf v1.35.1 // indirect + gopkg.in/go-playground/validator.v9 v9.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.28.4 // indirect - k8s.io/component-base v0.28.4 // indirect - k8s.io/klog/v2 v2.110.1 // indirect - k8s.io/kube-openapi v0.0.0-20231113174909-778a5567bc1e // indirect - knative.dev/networking v0.0.0-20231115015815-3af9769712cd // indirect + k8s.io/apiextensions-apiserver v0.31.0 // indirect + k8s.io/apiserver v0.31.0 // indirect + k8s.io/component-base v0.31.0 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20240827152857-f7e401e7b4c2 // indirect + knative.dev/networking v0.0.0-20240815142417-37fdbdd0854b // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) -replace ( - // Fixes CVE-2024-21626 - github.com/containerd/containerd => github.com/containerd/containerd v1.7.13 - // Fixes CVE-2022-21698 and CVE-2023-45142 - // this dependency comes from k8s.io/component-base@v0.26.4 and k8s.io/apiextensions-apiserver@v0.26.4 - // before removing it make sure that the next version of the related k8s dependencies contains the fix - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 - - // Fixes CVE-2023-45288 - golang.org/x/net => golang.org/x/net v0.23.0 - // can be removed when the indirect depdency is in the same version or higher - // Fixes CVE-2024-24786 - Infinite loop in JSON unmarshaling in google.golang.org/protobuf - google.golang.org/protobuf => google.golang.org/protobuf v1.33.0 - // Watch future versions of kserve where knative/serving will be updated. - // Fixes knative.dev/serving Uncontrolled Resource Consumption - // https://www.cve.org/CVERecord?id=CVE-2023-48713 - knative.dev/serving => knative.dev/serving v0.39.3 -) +replace github.com/imdario/mergo => github.com/imdario/mergo v0.3.5 diff --git a/go.sum b/go.sum index 60da66ad..8bc3f138 100644 --- a/go.sum +++ b/go.sum @@ -4,28 +4,42 @@ cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSR cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= -cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4= +cloud.google.com/go v0.115.1 h1:Jo0SM9cQnSkYfp44+v+NQXHpcHqlnRJk2qxh6yvxxxQ= +cloud.google.com/go v0.115.1/go.mod h1:DuujITeaufu3gL68/lOFIirVNJwQeyf5UXyi+Wbgknc= +cloud.google.com/go/auth v0.9.1 h1:+pMtLEV2k0AXKvs/tGZojuj6QaioxfUjOpMsG5Gtx+w= +cloud.google.com/go/auth v0.9.1/go.mod h1:Sw8ocT5mhhXxFklyhT12Eiy0ed6tTrPMCJjSI8KhYLk= +cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY= +cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY= cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc= -cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI= -cloud.google.com/go/storage v1.38.0 h1:Az68ZRGlnNTpIBbLjSMIV2BDcwwXYlRlQzis0llkpJg= -cloud.google.com/go/storage v1.38.0/go.mod h1:tlUADB0mAb9BgYls9lq+8MGkfzOXuLrnHXlpHmvFJoY= +cloud.google.com/go/iam v1.2.0 h1:kZKMKVNk/IsSSc/udOb83K0hL/Yh/Gcqpz+oAkoIFN8= +cloud.google.com/go/iam v1.2.0/go.mod h1:zITGuWgsLZxd8OwAlX+eMFgZDXzBm7icj1PVTYG766Q= +cloud.google.com/go/longrunning v0.6.0 h1:mM1ZmaNsQsnb+5n1DNPeL0KwQd9jQRqSqSDEkBZr+aI= +cloud.google.com/go/longrunning v0.6.0/go.mod h1:uHzSZqW89h7/pasCWNYdUpwGz3PcVWhrWupreVPYLts= +cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= +cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d h1:LblfooH1lKOpp1hIhukktmSAxFkqMPFk9KR6iZ0MJNI= contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d/go.mod h1:IshRmMJBhDfFj5Y67nVhMYTTIze91RUeT73ipWKs/GY= contrib.go.opencensus.io/exporter/prometheus v0.4.2 h1:sqfsYl5GIY/L570iT+l93ehxaWJs2/OwXtiWwew3oAg= contrib.go.opencensus.io/exporter/prometheus v0.4.2/go.mod h1:dvEHbiKmgvbr5pjaF9fpw1KeYcjrnC1J8B+JKjsZyRQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/aws/aws-sdk-go v1.48.0 h1:1SeJ8agckRDQvnSCt1dGZYAwUaoD2Ixj6IaXB4LCv8Q= -github.com/aws/aws-sdk-go v1.48.0/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= +github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/blendle/zapdriver v1.3.1 h1:C3dydBOWYRiOk+B8X9IVZ5IOe+7cl+tGOexN4QqHfpE= github.com/blendle/zapdriver v1.3.1/go.mod h1:mdXfREi6u5MArG4j9fewC+FGnXaBR+T4Ox4J2u4eHCc= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= @@ -33,27 +47,29 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= -github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= +github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= -github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.7.0 h1:nJqP7uwL84RJInrohHfW0Fx3awjbm8qZeFv0nW9SYGc= -github.com/evanphx/json-patch/v5 v5.7.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= +github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= @@ -61,24 +77,25 @@ github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBj github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= -github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= -github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -91,54 +108,59 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/cel-go v0.20.1 h1:nDx9r8S3L4pE61eDdt8igGj8rf5kjYR3ILxWIpWNi84= +github.com/google/cel-go v0.20.1/go.mod h1:kWcIzTsPX0zmQ+H3TirHstLLf9ep5QTsZBN9u4dOYLg= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-containerregistry v0.16.1 h1:rUEt426sR6nyrL3gt+18ibRcvYpKYdpsa5ZW7MA08dQ= -github.com/google/go-containerregistry v0.16.1/go.mod h1:u0qB2l7mvtWVR5kNcbFIhFY1hLbf8eeGapA+vbFDCtQ= +github.com/google/go-containerregistry v0.20.2 h1:B1wPJ1SN/S7pB+ZAimcciVD+r+yV/l/DSArMxlbwseo= +github.com/google/go-containerregistry v0.20.2/go.mod h1:z38EKdKh4h7IP2gSfUUqEvalZBqs6AoLeWfUy34nQC8= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= -github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= -github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSFBy+X1V0o+l+8NF1avt4HWl7cA= +github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= +github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= -github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/enterprise-certificate-proxy v0.3.3 h1:QRje2j5GZimBzlbhGA2V2QlGNgL8G6e+wGo/+/2bWI0= +github.com/googleapis/enterprise-certificate-proxy v0.3.3/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= -github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= +github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= +github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= github.com/googleapis/google-cloud-go-testing v0.0.0-20210719221736-1c9a4c676720 h1:zC34cGQu69FG7qzJ3WiKW244WfhDC3xxYMeNOX2gtUQ= github.com/googleapis/google-cloud-go-testing v0.0.0-20210719221736-1c9a4c676720/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1 h1:6UKoz5ujsI55KNpsJH3UwCq3T8kKbZwNZBNPuTTje8U= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1/go.mod h1:YvJ2f6MplWDhfxiUC3KpyTy76kYUZA4W3pTv/wdKQ9Y= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -147,10 +169,12 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= -github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 h1:liMMTbpW34dhU4az1GN0pTPADwNmvoRSeoZ6PItiqnY= +github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -160,23 +184,24 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kserve/kserve v0.12.1 h1:gVbrtEokm0VcBa3p1HP4i8aBgktH/WLRMBE6xFQHW3Y= -github.com/kserve/kserve v0.12.1/go.mod h1:Rz6mjJFdMIW12dy5enQF767P+xprmltC3+lL2HZwpMY= -github.com/kuadrant/authorino v0.15.0 h1:Xw/buh/wTINdL+IpLSxhlpet4hpleMxZzfx39c4VQng= -github.com/kuadrant/authorino v0.15.0/go.mod h1:vXkHKrntn8DR7kt8a8Ohxq+2lgAD0jWivThoP+7ASew= +github.com/kserve/kserve v0.14.0 h1:WvMrKJCKFrJoGGR2mNM5p7up4EvToec2IKqQeps2/wo= +github.com/kserve/kserve v0.14.0/go.mod h1:aecoWlJsJy++uLpntX1LUXUeyUD7OwGQ/0zoVP0K7CA= +github.com/kuadrant/authorino v0.18.1 h1:gIft1auKXHCsOiFTKeK4Psskq6sJJsxQUgB6YZYYrjg= +github.com/kuadrant/authorino v0.18.1/go.mod h1:70VTo3BuAgHFqWhQHEgjJwlDOKNcsmA0A3WebRQeGUA= github.com/kubeflow/model-registry v0.2.11 h1:DGVyns1Xk5qhgWjnvLsqPKTWNidXPstQx8eP+i1o4Z0= github.com/kubeflow/model-registry v0.2.11/go.mod h1:N9FYXwSPXdR1eAqYny2Nj4OhWOzbVC50iU0DOcidZRw= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -191,16 +216,16 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= -github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= +github.com/onsi/ginkgo/v2 v2.20.1 h1:YlVIbqct+ZmnEph770q9Q7NVAz4wwIiVNahee6JyUzo= +github.com/onsi/ginkgo/v2 v2.20.1/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= -github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8= +github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/openshift/api v3.9.0+incompatible h1:fJ/KsefYuZAjmrr3+5U9yZIZbTOpVkDDLDLFresAeYs= -github.com/openshift/api v3.9.0+incompatible/go.mod h1:dh9o4Fs58gpFXGSYfnVxGR9PnV53I8TW84pQaJDdGiY= +github.com/openshift/api v0.0.0-20240912201240-0a8800162826 h1:A8D9SN/hJUwAbdO0rPCVTqmuBOctdgurr53gK701SYo= +github.com/openshift/api v0.0.0-20240912201240-0a8800162826/go.mod h1:OOh6Qopf21pSzqNVCB5gomomBXb8o5sGKZxG2KNpaXM= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -209,21 +234,26 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.64.1 h1:bvntWler8vOjDJtxBwGDakGNC6srSZmgawGM9Jf7HC8= github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.64.1/go.mod h1:cfNgxpCPGyIydmt3HcwDqKDt0nYdlGRhzftl+DZH7WA= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_golang v1.20.2 h1:5ctymQzZlyOON1666svgwn3s6IKWgfbjsejTMiXIyjg= +github.com/prometheus/client_golang v1.20.2/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= -github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= -github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= -github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= -github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= -github.com/prometheus/statsd_exporter v0.25.0 h1:gpVF1TMf1UqMJmBDpzBYrEaGOFMpbMBYYYUDwM38Y/I= -github.com/prometheus/statsd_exporter v0.25.0/go.mod h1:HwzfSvg6ehmb0Qg71ZuFrlgj5XQt9C+MGVLz5Gt5lqc= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.57.0 h1:Ro/rKjwdq9mZn1K5QPctzh+MA4Lp0BuYk5ZZEVhoNcY= +github.com/prometheus/common v0.57.0/go.mod h1:7uRPFSUTbfZWsJ7MHY56sqt7hLQu3bxXHDnNhl8E9qI= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/prometheus/statsd_exporter v0.27.1 h1:tcRJOmwlA83HPfWzosAgr2+zEN5XDFv+M2mn/uYkn5Y= +github.com/prometheus/statsd_exporter v0.27.1/go.mod h1:vA6ryDfsN7py/3JApEst6nLTJboq66XsNcJGNmC88NQ= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -234,51 +264,58 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= -github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.3 h1:bwWLZU7icoKRG+C+0PNwIKC6FCJO/Q3p2pZvuP0jN94= +github.com/tidwall/gjson v1.17.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 h1:KfYpVmrjI7JuToy5k8XV3nkapjWx48k4E4JOtVstzQI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0/go.mod h1:SeQhzAEccGVZVEy7aH87Nh0km+utSpo1pTv6eMMop48= -go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= -go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= -go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= -go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= -go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= -go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= -go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= -go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= +go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= +go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 h1:qFffATk0X+HD+f1Z8lswGiOQYKHRlzfmdJm0wEaVrFA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0/go.mod h1:MOiCmryaYtc+V0Ei+Tx9o5S1ZjA7kzLucuVuyzBZloQ= +go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= +go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= +go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= +go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= +go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= +go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= -go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= -golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= +golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= +golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -288,12 +325,25 @@ golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -306,8 +356,6 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -322,30 +370,23 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -361,24 +402,20 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= +golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.171.0 h1:w174hnBPqut76FzW5Qaupt7zY8Kql6fiVjgys4f58sU= -google.golang.org/api v0.171.0/go.mod h1:Hnq5AHm4OTMt2BUVjael2CWZFD6vksJdWCWiUAmjC9o= +google.golang.org/api v0.195.0 h1:Ude4N8FvTKnnQJHU48RFI40jOBgIrL8Zqr3/QeST6yU= +google.golang.org/api v0.195.0/go.mod h1:DOGRWuv3P8TU8Lnz7uQc4hyNqrBpMtD9ppW3wBJurgc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -391,12 +428,12 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= -google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= -google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 h1:wKguEg1hsxI2/L3hUYrpo1RVi48K+uTyzKqprwLXsb8= -google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 h1:e7S5W7MGGLaSu8j3YjdezkZ+m1/Nm0uRVRMEMGk26Xs= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/genproto v0.0.0-20240827150818-7e3bb234dfed h1:4C4dbrVFtfIp3GXJdMX1Sj25mahfn5DywOo65/2ISQ8= +google.golang.org/genproto v0.0.0-20240827150818-7e3bb234dfed/go.mod h1:ICjniACoWvcDz8c8bOsHVKuuSGDJy1z5M4G0DM3HzTc= +google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed h1:3RgNmBoI9MZhsj3QxC+AP/qQhNwpCLOvYDYYsFrhFt0= +google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed h1:J6izYgfBXAI3xTKLgxzTmUltdYaLsuBxFCgDHWJ/eXg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -406,12 +443,27 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v9 v9.31.0 h1:bmXmP2RSNtFES+bn4uYuHT7iJFJv7Vj+an+ZQdDaD1M= +gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= @@ -431,37 +483,41 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -istio.io/api v1.21.5 h1:0ga5mb+ipPaCITpXZKP1Ixi0US085tRKH5cnzx6zvoI= -istio.io/api v1.21.5/go.mod h1:TFCMUCAHRjxBv1CsIsFCsYHPHi4axVI4vdIzVr8eFjY= -istio.io/client-go v1.21.5 h1:li1cbY+J8YM98aFPB1cudl0e8dgHhTPm4TRL8v+QqdM= -istio.io/client-go v1.21.5/go.mod h1:eC6qqf6Fw/NRc4iSFfSnPMrDOhfaFhlgPiT6v6d3RzU= -k8s.io/api v0.29.0 h1:NiCdQMY1QOp1H8lfRyeEf8eOwV6+0xA6XEE44ohDX2A= -k8s.io/api v0.29.0/go.mod h1:sdVmXoz2Bo/cb77Pxi71IPTSErEW32xa4aXwKH7gfBA= -k8s.io/apiextensions-apiserver v0.28.4 h1:AZpKY/7wQ8n+ZYDtNHbAJBb+N4AXXJvyZx6ww6yAJvU= -k8s.io/apiextensions-apiserver v0.28.4/go.mod h1:pgQIZ1U8eJSMQcENew/0ShUTlePcSGFq6dxSxf2mwPM= -k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o= -k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMeis= -k8s.io/client-go v0.29.0 h1:KmlDtFcrdUzOYrBhXHgKw5ycWzc3ryPX5mQe0SkG3y8= -k8s.io/client-go v0.29.0/go.mod h1:yLkXH4HKMAywcrD82KMSmfYg2DlE8mepPR4JGSo5n38= -k8s.io/component-base v0.28.4 h1:c/iQLWPdUgI90O+T9TeECg8o7N3YJTiuz2sKxILYcYo= -k8s.io/component-base v0.28.4/go.mod h1:m9hR0uvqXDybiGL2nf/3Lf0MerAfQXzkfWhUY58JUbU= -k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= -k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= -k8s.io/kube-openapi v0.0.0-20231113174909-778a5567bc1e h1:snPmy96t93RredGRjKfMFt+gvxuVAncqSAyBveJtr4Q= -k8s.io/kube-openapi v0.0.0-20231113174909-778a5567bc1e/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -knative.dev/networking v0.0.0-20231115015815-3af9769712cd h1:VDtYz+hybqIAEp8NM2tAi2QV4D8Cc5DWLoXLi5IcZjE= -knative.dev/networking v0.0.0-20231115015815-3af9769712cd/go.mod h1:HQ3rA7qrKVWvZUl6GGQefn/PzNXlX4e94KpbwBEjFcQ= -knative.dev/pkg v0.0.0-20231115001034-97c7258e3a98 h1:uvOLwp5Ar7oJlaYEszh51CemuZc1sRRI14xzKhUEF3U= -knative.dev/pkg v0.0.0-20231115001034-97c7258e3a98/go.mod h1:56Qcm0ai7xPWqGxpOnjRi4sAX9fZM9UDTk7fKyjUqZM= -knative.dev/serving v0.39.3 h1:x3p3iCY0eKwKZmlXUZfc9C0YawyiB6Kc1HlE66b530I= -knative.dev/serving v0.39.3/go.mod h1:bWylSgwnRZeL659qy7m3/TZioYk25TIfusPUEeR695A= -maistra.io/api v0.0.0-20230417135504-0536f6c22b1c h1:WNBqA7R23P/TDkzP/wa3mfE4Bd9eM8NzWiwhcNyWAgk= -maistra.io/api v0.0.0-20230417135504-0536f6c22b1c/go.mod h1:YdrOpeJBddUNHKIuhqlsNje9YUBFHl2pho7mhYwmsYs= +istio.io/api v1.23.0 h1:yqv3lNW6XSYS5XkbEkxsmFROXIQznp4lFWqj7xKEqCA= +istio.io/api v1.23.0/go.mod h1:QPSTGXuIQdnZFEm3myf9NZ5uBMwCdJWUvfj9ZZ+2oBM= +istio.io/client-go v1.23.0 h1://xojbifr84q29WE3eMx74p36hD4lvcejX1KxE3iJvY= +istio.io/client-go v1.23.0/go.mod h1:3qX/KBS5aR47QV4JhphcZl5ysnZ53x78TBjNQLM2TC4= +k8s.io/api v0.31.0 h1:b9LiSjR2ym/SzTOlfMHm1tr7/21aD7fSkqgD/CVJBCo= +k8s.io/api v0.31.0/go.mod h1:0YiFF+JfFxMM6+1hQei8FY8M7s1Mth+z/q7eF1aJkTE= +k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24kyLSk= +k8s.io/apiextensions-apiserver v0.31.0/go.mod h1:b9aMDEYaEe5sdK+1T0KU78ApR/5ZVp4i56VacZYEHxk= +k8s.io/apimachinery v0.31.0 h1:m9jOiSr3FoSSL5WO9bjm1n6B9KROYYgNZOb4tyZ1lBc= +k8s.io/apimachinery v0.31.0/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/apiserver v0.31.0 h1:p+2dgJjy+bk+B1Csz+mc2wl5gHwvNkC9QJV+w55LVrY= +k8s.io/apiserver v0.31.0/go.mod h1:KI9ox5Yu902iBnnyMmy7ajonhKnkeZYJhTZ/YI+WEMk= +k8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8= +k8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU= +k8s.io/component-base v0.31.0 h1:/KIzGM5EvPNQcYgwq5NwoQBaOlVFrghoVGr8lG6vNRs= +k8s.io/component-base v0.31.0/go.mod h1:TYVuzI1QmN4L5ItVdMSXKvH7/DtvIuas5/mm8YT3rTo= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240827152857-f7e401e7b4c2 h1:GKE9U8BH16uynoxQii0auTjmmmuZ3O0LFMN6S0lPPhI= +k8s.io/kube-openapi v0.0.0-20240827152857-f7e401e7b4c2/go.mod h1:coRQXBK9NxO98XUv3ZD6AK3xzHCxV6+b7lrquKwaKzA= +k8s.io/utils v0.0.0-20240821151609-f90d01438635 h1:2wThSvJoW/Ncn9TmQEYXRnevZXi2duqHWf5OX9S3zjI= +k8s.io/utils v0.0.0-20240821151609-f90d01438635/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +knative.dev/networking v0.0.0-20240815142417-37fdbdd0854b h1:ws/Jeho6on84+5tfNKLAKriVVGIwivHbgPEtZjBfcs0= +knative.dev/networking v0.0.0-20240815142417-37fdbdd0854b/go.mod h1:2eMQVGLBZ5Kj1C4kKPuPhO7BsUeF6fkmhZFDQPIP+88= +knative.dev/pkg v0.0.0-20240815051656-89743d9bbf7c h1:2crXVk4FG0dSG6WHaIT+WKbUzn7qG2wn0AfYmvA22zs= +knative.dev/pkg v0.0.0-20240815051656-89743d9bbf7c/go.mod h1:cI2RPEEHZk+/dBpfHobs0aBdPA1mMZVUVWnGAc8NSzM= +knative.dev/serving v0.42.2 h1:yKieg3MeNvpVz+4JJPbvmpee3v3LK3zO5h5HJBtzaNk= +knative.dev/serving v0.42.2/go.mod h1:3cgU8/864RcqA0ZPrc3jFcmS3uJL/mOlUZiYsXonwaE= +maistra.io/api v0.0.0-20240319144440-ffa91c765143 h1:MlOlBUSadkKmDgD7gTFOJ7kw5RJISopBpu0042Iv0iE= +maistra.io/api v0.0.0-20240319144440-ffa91c765143/go.mod h1:LzK0bOLCJAO7FIZ7dMdVHE0L9S6KqZt+BOHhGbi2hY0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= -sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 h1:2770sDpzrjjsAtVhSeUFseziht227YAWYHLGNM8QPwY= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= +sigs.k8s.io/controller-runtime v0.19.1 h1:Son+Q40+Be3QWb+niBXAg2vFiYWolDjjRfO8hn/cxOk= +sigs.k8s.io/controller-runtime v0.19.1/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt index 29c55ecd..ff72ff2a 100644 --- a/hack/boilerplate.go.txt +++ b/hack/boilerplate.go.txt @@ -1,5 +1,5 @@ /* -Copyright 2022. +Copyright 2024. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/hack/verify_nvidia_nim_api.go b/hack/verify_nvidia_nim_api.go index 9add39d4..248f0e13 100644 --- a/hack/verify_nvidia_nim_api.go +++ b/hack/verify_nvidia_nim_api.go @@ -20,9 +20,10 @@ import ( "encoding/json" "flag" "fmt" - "github.com/opendatahub-io/odh-model-controller/controllers/utils" "os" "strings" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" ) // Use this script for validating NVIDIA API access used by the NIM Account Controller. diff --git a/controllers/comparators/authconfig_comparator.go b/internal/controller/comparators/authconfig_comparator.go similarity index 100% rename from controllers/comparators/authconfig_comparator.go rename to internal/controller/comparators/authconfig_comparator.go diff --git a/controllers/comparators/clusterrolebinding_comparator.go b/internal/controller/comparators/clusterrolebinding_comparator.go similarity index 100% rename from controllers/comparators/clusterrolebinding_comparator.go rename to internal/controller/comparators/clusterrolebinding_comparator.go diff --git a/controllers/comparators/configmap_comparator.go b/internal/controller/comparators/configmap_comparator.go similarity index 100% rename from controllers/comparators/configmap_comparator.go rename to internal/controller/comparators/configmap_comparator.go diff --git a/controllers/comparators/gateway_comparator.go b/internal/controller/comparators/gateway_comparator.go similarity index 100% rename from controllers/comparators/gateway_comparator.go rename to internal/controller/comparators/gateway_comparator.go diff --git a/controllers/comparators/inferenceservice_comparator.go b/internal/controller/comparators/inferenceservice_comparator.go similarity index 95% rename from controllers/comparators/inferenceservice_comparator.go rename to internal/controller/comparators/inferenceservice_comparator.go index eafb70f8..244a7496 100644 --- a/controllers/comparators/inferenceservice_comparator.go +++ b/internal/controller/comparators/inferenceservice_comparator.go @@ -19,7 +19,7 @@ import ( "reflect" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - "github.com/opendatahub-io/odh-model-controller/controllers/constants" + "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/controllers/comparators/networkpolicy_comparator.go b/internal/controller/comparators/networkpolicy_comparator.go similarity index 96% rename from controllers/comparators/networkpolicy_comparator.go rename to internal/controller/comparators/networkpolicy_comparator.go index 3b0717ee..aa23286c 100644 --- a/controllers/comparators/networkpolicy_comparator.go +++ b/internal/controller/comparators/networkpolicy_comparator.go @@ -16,8 +16,9 @@ limitations under the License. package comparators import ( - "k8s.io/api/networking/v1" "reflect" + + v1 "k8s.io/api/networking/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/controllers/comparators/peerAuthentication_comparator.go b/internal/controller/comparators/peerAuthentication_comparator.go similarity index 99% rename from controllers/comparators/peerAuthentication_comparator.go rename to internal/controller/comparators/peerAuthentication_comparator.go index 251f766f..fc4a486b 100644 --- a/controllers/comparators/peerAuthentication_comparator.go +++ b/internal/controller/comparators/peerAuthentication_comparator.go @@ -16,8 +16,9 @@ limitations under the License. package comparators import ( - istiosecv1beta1 "istio.io/client-go/pkg/apis/security/v1beta1" "reflect" + + istiosecv1beta1 "istio.io/client-go/pkg/apis/security/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/controllers/comparators/podmonitor_comparator.go b/internal/controller/comparators/podmonitor_comparator.go similarity index 99% rename from controllers/comparators/podmonitor_comparator.go rename to internal/controller/comparators/podmonitor_comparator.go index e36f60ae..cf2bcf6f 100644 --- a/controllers/comparators/podmonitor_comparator.go +++ b/internal/controller/comparators/podmonitor_comparator.go @@ -16,8 +16,9 @@ limitations under the License. package comparators import ( - v1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "reflect" + + v1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/controllers/comparators/resourcecomparator.go b/internal/controller/comparators/resourcecomparator.go similarity index 100% rename from controllers/comparators/resourcecomparator.go rename to internal/controller/comparators/resourcecomparator.go diff --git a/controllers/comparators/rolebinding_comparator.go b/internal/controller/comparators/rolebinding_comparator.go similarity index 99% rename from controllers/comparators/rolebinding_comparator.go rename to internal/controller/comparators/rolebinding_comparator.go index e1bca166..540dd49f 100644 --- a/controllers/comparators/rolebinding_comparator.go +++ b/internal/controller/comparators/rolebinding_comparator.go @@ -16,8 +16,9 @@ limitations under the License. package comparators import ( - v1 "k8s.io/api/rbac/v1" "reflect" + + v1 "k8s.io/api/rbac/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/controllers/comparators/route_comparator.go b/internal/controller/comparators/route_comparator.go similarity index 99% rename from controllers/comparators/route_comparator.go rename to internal/controller/comparators/route_comparator.go index 9c1fa4b7..060098be 100644 --- a/controllers/comparators/route_comparator.go +++ b/internal/controller/comparators/route_comparator.go @@ -16,8 +16,9 @@ limitations under the License. package comparators import ( - v1 "github.com/openshift/api/route/v1" "reflect" + + v1 "github.com/openshift/api/route/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/controllers/comparators/service_comparator.go b/internal/controller/comparators/service_comparator.go similarity index 99% rename from controllers/comparators/service_comparator.go rename to internal/controller/comparators/service_comparator.go index f4032d1a..cbd60107 100644 --- a/controllers/comparators/service_comparator.go +++ b/internal/controller/comparators/service_comparator.go @@ -16,8 +16,9 @@ limitations under the License. package comparators import ( - v1 "k8s.io/api/core/v1" "reflect" + + v1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/controllers/comparators/serviceaccount_comparator.go b/internal/controller/comparators/serviceaccount_comparator.go similarity index 100% rename from controllers/comparators/serviceaccount_comparator.go rename to internal/controller/comparators/serviceaccount_comparator.go diff --git a/controllers/comparators/servicemeshmember_comparator.go b/internal/controller/comparators/servicemeshmember_comparator.go similarity index 100% rename from controllers/comparators/servicemeshmember_comparator.go rename to internal/controller/comparators/servicemeshmember_comparator.go diff --git a/controllers/comparators/servicemonitor_comparator.go b/internal/controller/comparators/servicemonitor_comparator.go similarity index 99% rename from controllers/comparators/servicemonitor_comparator.go rename to internal/controller/comparators/servicemonitor_comparator.go index 2ee07a60..5c546785 100644 --- a/controllers/comparators/servicemonitor_comparator.go +++ b/internal/controller/comparators/servicemonitor_comparator.go @@ -16,8 +16,9 @@ limitations under the License. package comparators import ( - v1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "reflect" + + v1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/controllers/comparators/telemetry_comparator.go b/internal/controller/comparators/telemetry_comparator.go similarity index 99% rename from controllers/comparators/telemetry_comparator.go rename to internal/controller/comparators/telemetry_comparator.go index 108073e6..d460f60b 100644 --- a/controllers/comparators/telemetry_comparator.go +++ b/internal/controller/comparators/telemetry_comparator.go @@ -16,8 +16,9 @@ limitations under the License. package comparators import ( - telemetryv1alpha1 "istio.io/client-go/pkg/apis/telemetry/v1alpha1" "reflect" + + telemetryv1alpha1 "istio.io/client-go/pkg/apis/telemetry/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/controllers/constants/constants.go b/internal/controller/constants/constants.go similarity index 100% rename from controllers/constants/constants.go rename to internal/controller/constants/constants.go diff --git a/controllers/constants/runtime-metrics.go b/internal/controller/constants/runtime-metrics.go similarity index 99% rename from controllers/constants/runtime-metrics.go rename to internal/controller/constants/runtime-metrics.go index b09b2572..1644ad26 100644 --- a/controllers/constants/runtime-metrics.go +++ b/internal/controller/constants/runtime-metrics.go @@ -16,7 +16,7 @@ limitations under the License. package constants const ( - //Caikit Standalone + // Caikit Standalone CaikitMetricsData = `{ "config": [ { @@ -70,7 +70,7 @@ const ( ] }` - //OpenVino Model Server + // OpenVino Model Server OvmsMetricsData = `{ "config": [ { @@ -178,7 +178,7 @@ const ( ] }` - //vLLM + // vLLM VllmMetricsData = `{ "config": [ { diff --git a/controllers/kserve_customcacert_controller.go b/internal/controller/core/configmap_controller.go similarity index 83% rename from controllers/kserve_customcacert_controller.go rename to internal/controller/core/configmap_controller.go index 8108e1d8..7c87152a 100644 --- a/controllers/kserve_customcacert_controller.go +++ b/internal/controller/core/configmap_controller.go @@ -1,4 +1,5 @@ /* +Copyright 2024. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -13,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controllers +package core import ( "context" @@ -21,16 +22,19 @@ import ( "strings" "github.com/go-logr/logr" - "github.com/opendatahub-io/odh-model-controller/controllers/constants" - "github.com/opendatahub-io/odh-model-controller/controllers/utils" corev1 "k8s.io/api/core/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" ) const ( @@ -39,13 +43,16 @@ const ( odhGlobalCACertConfigMapName = constants.ODHGlobalCertConfigMapName ) -type KServeCustomCACertReconciler struct { +// ConfigMapReconciler was formerly known as KServeCustomCACertReconciler. +type ConfigMapReconciler struct { client.Client - Log logr.Logger + Scheme *runtime.Scheme } +// +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete + // reconcileConfigMap watch odh global ca cert and it will create/update/delete kserve custom cert configmap -func (r *KServeCustomCACertReconciler) reconcileConfigMap(configmap *corev1.ConfigMap, ctx context.Context, log logr.Logger) error { +func (r *ConfigMapReconciler) reconcileConfigMap(configmap *corev1.ConfigMap, ctx context.Context, log logr.Logger) error { var odhCustomCertData string // If kserve custom cert configmap changed, rollback it @@ -103,9 +110,9 @@ func reconcileOpenDataHubGlobalCACertConfigMap() predicate.Predicate { } } -func (r *KServeCustomCACertReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { +func (r *ConfigMapReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { // Initialize logger format - log := r.Log.WithValues("ConfigMap", req.Name, "namespace", req.Namespace) + log := log.FromContext(ctx).WithValues("ConfigMap", req.Name, "namespace", req.Namespace) configmap := &corev1.ConfigMap{} err := r.Get(ctx, req.NamespacedName, configmap) @@ -125,16 +132,13 @@ func (r *KServeCustomCACertReconciler) Reconcile(ctx context.Context, req ctrl.R } // SetupWithManager sets up the controller with the Manager. -func (r *KServeCustomCACertReconciler) SetupWithManager(mgr ctrl.Manager) error { +func (r *ConfigMapReconciler) SetupWithManager(mgr ctrl.Manager) error { // Create a builder that only watch OpenDataHub global certificate ConfigMap - builder := ctrl.NewControllerManagedBy(mgr). + return ctrl.NewControllerManagedBy(mgr). For(&corev1.ConfigMap{}). - WithEventFilter(reconcileOpenDataHubGlobalCACertConfigMap()) - err := builder.Complete(r) - if err != nil { - return err - } - return nil + Named("core-configmap"). + WithEventFilter(reconcileOpenDataHubGlobalCACertConfigMap()). + Complete(r) } func getDesiredCaCertConfigMapForKServe(configmapName string, namespace string, caCertData map[string]string) *corev1.ConfigMap { @@ -150,7 +154,7 @@ func getDesiredCaCertConfigMapForKServe(configmapName string, namespace string, return desiredConfigMap } -func (r *KServeCustomCACertReconciler) processDelta(ctx context.Context, log logr.Logger, desiredConfigMap *corev1.ConfigMap, existingConfigMap *corev1.ConfigMap) (err error) { +func (r *ConfigMapReconciler) processDelta(ctx context.Context, log logr.Logger, desiredConfigMap *corev1.ConfigMap, existingConfigMap *corev1.ConfigMap) (err error) { hasChanged := false if isAdded(desiredConfigMap, existingConfigMap) { @@ -195,7 +199,7 @@ func (r *KServeCustomCACertReconciler) processDelta(ctx context.Context, log log } // This section is intended for regenerating StorageSecret using new data. -func (r *KServeCustomCACertReconciler) deleteStorageSecret(ctx context.Context, namespace string) error { +func (r *ConfigMapReconciler) deleteStorageSecret(ctx context.Context, namespace string) error { foundStorageSecret := &corev1.Secret{} err := r.Get(ctx, types.NamespacedName{ diff --git a/controllers/kserve_customcacert_controller_test.go b/internal/controller/core/configmap_controller_test.go similarity index 78% rename from controllers/kserve_customcacert_controller_test.go rename to internal/controller/core/configmap_controller_test.go index 08ce52ac..cb798235 100644 --- a/controllers/kserve_customcacert_controller_test.go +++ b/internal/controller/core/configmap_controller_test.go @@ -1,4 +1,5 @@ /* +Copyright 2024. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -13,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controllers +package core import ( "context" @@ -23,10 +24,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/opendatahub-io/odh-model-controller/controllers/constants" corev1 "k8s.io/api/core/v1" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" ) const ( @@ -34,7 +36,7 @@ const ( kserveCustomCACustomBundleConfigMapUpdatedPath = "./testdata/configmaps/odh-kserve-custom-ca-cert-configmap-updated.yaml" ) -var _ = Describe("KServe Custom CA Cert ConfigMap controller", func() { +var _ = Describe("KServe Custom CA Cert ConfigMap Controller", func() { ctx := context.Background() AfterEach(func() { @@ -45,7 +47,7 @@ var _ = Describe("KServe Custom CA Cert ConfigMap controller", func() { }, } - Expect(cli.Delete(ctx, configmap)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, configmap)).Should(Succeed()) }) Context("when a configmap 'odh-trusted-ca-bundle' exists", func() { @@ -54,9 +56,9 @@ var _ = Describe("KServe Custom CA Cert ConfigMap controller", func() { odhtrustedcacertConfigMap := &corev1.ConfigMap{} err := convertToStructuredResource(odhtrustedcabundleConfigMapPath, odhtrustedcacertConfigMap) Expect(err).NotTo(HaveOccurred()) - Expect(cli.Create(ctx, odhtrustedcacertConfigMap)).Should(Succeed()) + Expect(k8sClient.Create(ctx, odhtrustedcacertConfigMap)).Should(Succeed()) - _, err = waitForConfigMap(cli, WorkingNamespace, constants.KServeCACertConfigMapName, 30, 1*time.Second) + _, err = waitForConfigMap(k8sClient, WorkingNamespace, constants.KServeCACertConfigMapName, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) }) }) @@ -67,20 +69,20 @@ var _ = Describe("KServe Custom CA Cert ConfigMap controller", func() { odhtrustedcacertConfigMap := &corev1.ConfigMap{} err := convertToStructuredResource(odhtrustedcabundleConfigMapPath, odhtrustedcacertConfigMap) Expect(err).NotTo(HaveOccurred()) - Expect(cli.Create(ctx, odhtrustedcacertConfigMap)).Should(Succeed()) + Expect(k8sClient.Create(ctx, odhtrustedcacertConfigMap)).Should(Succeed()) - _, err = waitForConfigMap(cli, WorkingNamespace, constants.KServeCACertConfigMapName, 30, 1*time.Second) + _, err = waitForConfigMap(k8sClient, WorkingNamespace, constants.KServeCACertConfigMapName, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) By("updating odh-trusted-ca-bundle configmap") updatedOdhtrustedcacertConfigMap := &corev1.ConfigMap{} err = convertToStructuredResource(odhtrustedcabundleConfigMapUpdatedPath, updatedOdhtrustedcacertConfigMap) Expect(err).NotTo(HaveOccurred()) - Expect(cli.Update(ctx, updatedOdhtrustedcacertConfigMap)).Should(Succeed()) + Expect(k8sClient.Update(ctx, updatedOdhtrustedcacertConfigMap)).Should(Succeed()) // Wait for updating ConfigMap time.Sleep(1 * time.Second) - kserveCACertConfigmap, err := waitForConfigMap(cli, WorkingNamespace, constants.KServeCACertConfigMapName, 30, 1*time.Second) + kserveCACertConfigmap, err := waitForConfigMap(k8sClient, WorkingNamespace, constants.KServeCACertConfigMapName, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) expectedKserveCACertConfigmap := &corev1.ConfigMap{} err = convertToStructuredResource(kserveCustomCACustomBundleConfigMapUpdatedPath, expectedKserveCACertConfigmap) diff --git a/controllers/storageconfig_controller.go b/internal/controller/core/secret_controller.go similarity index 76% rename from controllers/storageconfig_controller.go rename to internal/controller/core/secret_controller.go index d3efbdcc..79007805 100644 --- a/controllers/storageconfig_controller.go +++ b/internal/controller/core/secret_controller.go @@ -1,4 +1,5 @@ /* +Copyright 2024. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -13,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controllers +package core import ( "context" @@ -21,26 +22,34 @@ import ( "reflect" "github.com/go-logr/logr" - "github.com/opendatahub-io/odh-model-controller/controllers/constants" + corev1 "k8s.io/api/core/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/retry" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" ) const ( storageSecretName = constants.DefaultStorageConfig ) -type StorageSecretReconciler struct { +// SecretReconciler reconciles a Secret object. Formerly +// known as StorageSecretReconciler +type SecretReconciler struct { client.Client - Log logr.Logger + Scheme *runtime.Scheme } +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete + // newStorageSecret takes a list of data connection secrets and generates a single storage config secret // https://github.com/kserve/modelmesh-serving/blob/main/docs/predictors/setup-storage.md func newStorageSecret(dataConnectionSecretsList *corev1.SecretList, odhCustomCertData string) *corev1.Secret { @@ -77,10 +86,10 @@ func CompareStorageSecrets(s1 corev1.Secret, s2 corev1.Secret) bool { // reconcileSecret grabs all data connection secrets in the triggering namespace and // creates/updates the storage config secret -func (r *StorageSecretReconciler) reconcileSecret(secret *corev1.Secret, +func (r *SecretReconciler) reconcileSecret(secret *corev1.Secret, ctx context.Context, newStorageSecret func(dataConnectionSecretsList *corev1.SecretList, odhCustomCertData string) *corev1.Secret) error { // Initialize logger format - log := r.Log.WithValues("secret", secret.Name, "namespace", secret.Namespace) + logger := log.FromContext(ctx) // Grab all data connections in the namespace dataConnectionSecretsList := &corev1.SecretList{} @@ -93,9 +102,9 @@ func (r *StorageSecretReconciler) reconcileSecret(secret *corev1.Secret, return err } if len(dataConnectionSecretsList.Items) == 0 { - log.Info("No data connections found in namespace") - if err := r.deleteStorageSecret(ctx, secret.Namespace, log); err != nil { - log.Error(err, "Failed to delete the storage-config secret") + logger.Info("No data connections found in namespace") + if err := r.deleteStorageSecret(ctx, secret.Namespace, logger); err != nil { + logger.Error(err, "Failed to delete the storage-config secret") return err } return nil @@ -110,7 +119,7 @@ func (r *StorageSecretReconciler) reconcileSecret(secret *corev1.Secret, if err != nil { if apierrs.IsNotFound(err) { - log.Info("unable to fetch the ODH Global Cert ConfigMap", "error", err) + logger.Info("unable to fetch the ODH Global Cert ConfigMap", "error", err) } else { return err } @@ -133,24 +142,24 @@ func (r *StorageSecretReconciler) reconcileSecret(secret *corev1.Secret, }, foundStorageSecret) if err != nil { if apierrs.IsNotFound(err) { - log.Info("Creating Storage Config Secret") + logger.Info("Creating Storage Config Secret") // Create the storage config secret if it doesn't already exist err = r.Create(ctx, desiredStorageSecret) if err != nil && !apierrs.IsAlreadyExists(err) { - log.Error(err, "Unable to create the Storage Config Secret") + logger.Error(err, "Unable to create the Storage Config Secret") return err } justCreated = true } else { - log.Error(err, "Unable to fetch the Storage Config Secret") + logger.Error(err, "Unable to fetch the Storage Config Secret") return err } } // Reconcile the Storage Config Secret if it has been manually modified if !justCreated && !CompareStorageSecrets(*desiredStorageSecret, *foundStorageSecret) { - log.Info("Reconciling Storage Config Secret") + logger.Info("Reconciling Storage Config Secret") err := retry.RetryOnConflict(retry.DefaultRetry, func() error { // Get the last Storage Config revision @@ -166,7 +175,7 @@ func (r *StorageSecretReconciler) reconcileSecret(secret *corev1.Secret, return r.Update(ctx, foundStorageSecret) }) if err != nil { - log.Error(err, "Unable to reconcile the Storage Config Secret") + logger.Error(err, "Unable to reconcile the Storage Config Secret") return err } } @@ -175,7 +184,7 @@ func (r *StorageSecretReconciler) reconcileSecret(secret *corev1.Secret, } // ReconcileStorageSecret will manage the creation, update and deletion of the Storage Config Secret -func (r *StorageSecretReconciler) ReconcileStorageSecret( +func (r *SecretReconciler) ReconcileStorageSecret( secret *corev1.Secret, ctx context.Context) error { return r.reconcileSecret(secret, ctx, newStorageSecret) } @@ -202,17 +211,27 @@ func reconcileOpenDataHubSecrets() predicate.Predicate { } } -func (r *StorageSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the Secret object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/reconcile +func (r *SecretReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { // Initialize logger format - log := r.Log.WithValues("Secret", req.Name, "namespace", req.Namespace) + logger := log.FromContext(ctx).WithValues("Secret", req.Name, "namespace", req.Namespace) + ctx = log.IntoContext(ctx, logger) secret := &corev1.Secret{} err := r.Get(ctx, req.NamespacedName, secret) if err != nil && apierrs.IsNotFound(err) { - log.Info("Data Connection not found") + logger.Info("Data Connection not found") secret.Namespace = req.Namespace } else if err != nil { - log.Error(err, "Unable to fetch the secret") + logger.Error(err, "Unable to fetch the secret") return ctrl.Result{}, err } @@ -224,19 +243,15 @@ func (r *StorageSecretReconciler) Reconcile(ctx context.Context, req ctrl.Reques } // SetupWithManager sets up the controller with the Manager. -func (r *StorageSecretReconciler) SetupWithManager(mgr ctrl.Manager) error { - // Create a builder that only watch secrets that have the Open Data Hub label on them - builder := ctrl.NewControllerManagedBy(mgr). +func (r *SecretReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). For(&corev1.Secret{}). - WithEventFilter(reconcileOpenDataHubSecrets()) - err := builder.Complete(r) - if err != nil { - return err - } - return nil + Named("secret"). + WithEventFilter(reconcileOpenDataHubSecrets()). + Complete(r) } -func (r *StorageSecretReconciler) deleteStorageSecret(ctx context.Context, namespace string, log logr.Logger) error { +func (r *SecretReconciler) deleteStorageSecret(ctx context.Context, namespace string, log logr.Logger) error { foundStorageSecret := &corev1.Secret{} err := r.Get(ctx, types.NamespacedName{ diff --git a/controllers/storageconfig_controller_test.go b/internal/controller/core/secret_controller_test.go similarity index 75% rename from controllers/storageconfig_controller_test.go rename to internal/controller/core/secret_controller_test.go index b752ce15..b102ba97 100644 --- a/controllers/storageconfig_controller_test.go +++ b/internal/controller/core/secret_controller_test.go @@ -1,4 +1,5 @@ /* +Copyright 2024. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -13,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controllers +package core import ( "context" @@ -22,13 +23,13 @@ import ( "reflect" "time" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/opendatahub-io/odh-model-controller/controllers/constants" - apierrs "k8s.io/apimachinery/pkg/api/errors" + "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" ) const ( @@ -39,9 +40,7 @@ const ( storageconfigUpdatedCertEncodedPath = "./testdata/secrets/storageconfig-updated-cert-encoded.yaml" ) -var _ = Describe("StorageConfig controller", func() { - ctx := context.Background() - +var _ = Describe("Secret Controller (StorageConfig controller)", func() { Context("when a dataconnection secret that has 'opendatahub.io/managed=true' and 'opendatahub.io/dashboard=true' is created", func() { It("should create a storage-config secret", func() { dataconnectionStringSecret := &corev1.Secret{} @@ -49,9 +48,9 @@ var _ = Describe("StorageConfig controller", func() { By("creating dataconnection secret") err := convertToStructuredResource(dataconnectionStringPath, dataconnectionStringSecret) Expect(err).NotTo(HaveOccurred()) - Expect(cli.Create(ctx, dataconnectionStringSecret)).Should(Succeed()) + Expect(k8sClient.Create(ctx, dataconnectionStringSecret)).Should(Succeed()) - storegeconfigSecret, err := waitForSecret(cli, WorkingNamespace, constants.DefaultStorageConfig, 30, 1*time.Second) + storegeconfigSecret, err := waitForSecret(k8sClient, WorkingNamespace, constants.DefaultStorageConfig, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) expectedStorageConfigSecret := &corev1.Secret{} @@ -68,17 +67,17 @@ var _ = Describe("StorageConfig controller", func() { By("creating dataconnection secret") err := convertToStructuredResource(dataconnectionStringPath, dataconnectionStringSecret) Expect(err).NotTo(HaveOccurred()) - Expect(cli.Create(ctx, dataconnectionStringSecret)).Should(Succeed()) + Expect(k8sClient.Create(ctx, dataconnectionStringSecret)).Should(Succeed()) - storegeconfigSecret, err := waitForSecret(cli, WorkingNamespace, constants.DefaultStorageConfig, 30, 1*time.Second) + storegeconfigSecret, err := waitForSecret(k8sClient, WorkingNamespace, constants.DefaultStorageConfig, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) Expect(storegeconfigSecret).NotTo(BeNil()) By("deleting the dataconnection secret") Expect(err).NotTo(HaveOccurred()) - Expect(cli.Delete(ctx, dataconnectionStringSecret)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, dataconnectionStringSecret)).Should(Succeed()) - storegeconfigSecret, err = waitForSecret(cli, WorkingNamespace, constants.DefaultStorageConfig, 30, 1*time.Second) + storegeconfigSecret, err = waitForSecret(k8sClient, WorkingNamespace, constants.DefaultStorageConfig, 30, 1*time.Second) Expect(storegeconfigSecret).To(BeNil()) Expect(err).To(HaveOccurred()) Expect(err).To(BeAssignableToTypeOf(&apierrs.StatusError{})) @@ -90,23 +89,23 @@ var _ = Describe("StorageConfig controller", func() { By("creating dataconnection secret") err := convertToStructuredResource(dataconnectionStringPath, dataconnectionStringSecret) Expect(err).NotTo(HaveOccurred()) - Expect(cli.Create(ctx, dataconnectionStringSecret)).Should(Succeed()) + Expect(k8sClient.Create(ctx, dataconnectionStringSecret)).Should(Succeed()) - storegeconfigSecret, err := waitForSecret(cli, WorkingNamespace, constants.DefaultStorageConfig, 30, 1*time.Second) + storegeconfigSecret, err := waitForSecret(k8sClient, WorkingNamespace, constants.DefaultStorageConfig, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) Expect(storegeconfigSecret).NotTo(BeNil()) By("updating storage-config label opendatahub.io/managed: false") - err = updateSecretLabel(cli, WorkingNamespace, storageSecretName, "opendatahub.io/managed", "false") + err = updateSecretLabel(k8sClient, WorkingNamespace, storageSecretName, "opendatahub.io/managed", "false") Expect(err).NotTo(HaveOccurred()) - _, err = waitForSecret(cli, WorkingNamespace, storageSecretName, 30, 1*time.Second) + _, err = waitForSecret(k8sClient, WorkingNamespace, storageSecretName, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) By("deleting the dataconnection secret") Expect(err).NotTo(HaveOccurred()) - Expect(cli.Delete(ctx, dataconnectionStringSecret)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, dataconnectionStringSecret)).Should(Succeed()) - storegeconfigSecret, err = waitForSecret(cli, WorkingNamespace, constants.DefaultStorageConfig, 30, 1*time.Second) + storegeconfigSecret, err = waitForSecret(k8sClient, WorkingNamespace, constants.DefaultStorageConfig, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) Expect(storegeconfigSecret).NotTo(BeNil()) }) @@ -119,21 +118,21 @@ var _ = Describe("StorageConfig controller", func() { By("creating dataconnection secret") err := convertToStructuredResource(dataconnectionStringPath, dataconnectionStringSecret) Expect(err).NotTo(HaveOccurred()) - Expect(cli.Create(ctx, dataconnectionStringSecret)).Should(Succeed()) + Expect(k8sClient.Create(ctx, dataconnectionStringSecret)).Should(Succeed()) - storageconfigSecret := &corev1.Secret{} - _, err = waitForSecret(cli, WorkingNamespace, constants.DefaultStorageConfig, 30, 1*time.Second) + var storageconfigSecret *corev1.Secret + _, err = waitForSecret(k8sClient, WorkingNamespace, constants.DefaultStorageConfig, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) - err = updateSecretLabel(cli, WorkingNamespace, storageSecretName, "opendatahub.io/managed", "false") + err = updateSecretLabel(k8sClient, WorkingNamespace, storageSecretName, "opendatahub.io/managed", "false") Expect(err).NotTo(HaveOccurred()) - _, err = waitForSecret(cli, WorkingNamespace, storageSecretName, 30, 1*time.Second) + _, err = waitForSecret(k8sClient, WorkingNamespace, storageSecretName, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) - err = updateSecretData(cli, WorkingNamespace, storageSecretName, "aws-connection-minio", "unmanaged") + err = updateSecretData(k8sClient, WorkingNamespace, storageSecretName, "aws-connection-minio", "unmanaged") Expect(err).NotTo(HaveOccurred()) - storageconfigSecret, err = waitForSecret(cli, WorkingNamespace, storageSecretName, 30, 1*time.Second) + storageconfigSecret, err = waitForSecret(k8sClient, WorkingNamespace, storageSecretName, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) expectedStorageConfigSecret := &corev1.Secret{} @@ -151,14 +150,14 @@ var _ = Describe("StorageConfig controller", func() { odhKserveCustomCABundleConfigmap := &corev1.ConfigMap{} err := convertToStructuredResource(odhKserveCustomCABundleConfigMapPath, odhKserveCustomCABundleConfigmap) Expect(err).NotTo(HaveOccurred()) - Expect(cli.Create(ctx, odhKserveCustomCABundleConfigmap)).Should(Succeed()) + Expect(k8sClient.Create(ctx, odhKserveCustomCABundleConfigmap)).Should(Succeed()) By("creating dataconnection secret") err = convertToStructuredResource(dataconnectionStringPath, dataconnectionStringSecret) Expect(err).NotTo(HaveOccurred()) - Expect(cli.Create(ctx, dataconnectionStringSecret)).Should(Succeed()) + Expect(k8sClient.Create(ctx, dataconnectionStringSecret)).Should(Succeed()) - storageconfigSecret, err := waitForSecret(cli, WorkingNamespace, constants.DefaultStorageConfig, 30, 1*time.Second) + storageconfigSecret, err := waitForSecret(k8sClient, WorkingNamespace, constants.DefaultStorageConfig, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) // Check storage-config secret @@ -171,14 +170,14 @@ var _ = Describe("StorageConfig controller", func() { updatedOdhKserveCustomCABundleConfigmap := &corev1.ConfigMap{} err = convertToStructuredResource(kserveCustomCACustomBundleConfigMapUpdatedPath, updatedOdhKserveCustomCABundleConfigmap) Expect(err).NotTo(HaveOccurred()) - Expect(cli.Update(ctx, updatedOdhKserveCustomCABundleConfigmap)).Should(Succeed()) + Expect(k8sClient.Update(ctx, updatedOdhKserveCustomCABundleConfigmap)).Should(Succeed()) // Delete existing storage-config secret // This will be done by kserve_customcacert_controller but for this test, it needs to be delete manully to update the storage-config - Expect(cli.Delete(ctx, storageconfigSecret)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, storageconfigSecret)).Should(Succeed()) // Check updated storage-config secret - updatedStorageconfigSecret, err := waitForSecret(cli, WorkingNamespace, constants.DefaultStorageConfig, 30, 3*time.Second) + updatedStorageconfigSecret, err := waitForSecret(k8sClient, WorkingNamespace, constants.DefaultStorageConfig, 30, 3*time.Second) Expect(err).NotTo(HaveOccurred()) expectedUpdatedStorageConfigSecret := &corev1.Secret{} err = convertToStructuredResource(storageconfigUpdatedCertEncodedPath, expectedUpdatedStorageConfigSecret) @@ -195,9 +194,9 @@ var _ = Describe("StorageConfig controller", func() { By("creating data connection secret") err := convertToStructuredResource(dataconnectionStringPath, dataconnectionStringSecret) Expect(err).NotTo(HaveOccurred()) - Expect(cli.Create(ctx, dataconnectionStringSecret)).Should(Succeed()) + Expect(k8sClient.Create(ctx, dataconnectionStringSecret)).Should(Succeed()) - storageconfigSecret, err := waitForSecret(cli, WorkingNamespace, constants.DefaultStorageConfig, 30, 1*time.Second) + storageconfigSecret, err := waitForSecret(k8sClient, WorkingNamespace, constants.DefaultStorageConfig, 30, 1*time.Second) Expect(err).NotTo(HaveOccurred()) // Check storage-config secret @@ -219,7 +218,6 @@ var _ = Describe("StorageConfig controller", func() { Expect(compareSecrets(storageconfigSecret, expectedStorageConfigSecret)).Should((BeTrue())) }) }) - }) func updateSecretData(cli client.Client, namespace, secretName string, dataKey string, dataValue string) error { @@ -269,8 +267,11 @@ func updateSecretLabel(cli client.Client, namespace, secretName string, labelKey return nil } -func waitForSecret(cli client.Client, namespace, secretName string, maxTries int, delay time.Duration) (*corev1.Secret, error) { +func waitForSecret(cli client.Client, _, _ string, _ int, delay time.Duration) (*corev1.Secret, error) { time.Sleep(delay) + namespace := WorkingNamespace + secretName := constants.DefaultStorageConfig + maxTries := 30 ctx := context.Background() secret := &corev1.Secret{} diff --git a/internal/controller/core/suite_test.go b/internal/controller/core/suite_test.go new file mode 100644 index 00000000..748c9421 --- /dev/null +++ b/internal/controller/core/suite_test.go @@ -0,0 +1,216 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package core + +import ( + "context" + "fmt" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + kservev1alpha1 "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" + kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" + authorinov1beta2 "github.com/kuadrant/authorino/api/v1beta2" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + routev1 "github.com/openshift/api/route/v1" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + istioclientv1beta1 "istio.io/client-go/pkg/apis/networking/v1beta1" + istiov1beta1 "istio.io/client-go/pkg/apis/security/v1beta1" + corev1 "k8s.io/api/core/v1" + k8srbacv1 "k8s.io/api/rbac/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + k8sRuntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + // +kubebuilder:scaffold:imports + + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" + testutils "github.com/opendatahub-io/odh-model-controller/test/utils" +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var cfg *rest.Config +var k8sClient client.Client +var testEnv *envtest.Environment +var ctx context.Context +var cancel context.CancelFunc + +const ( + WorkingNamespace = "default" + odhtrustedcabundleConfigMapPath = "./testdata/configmaps/odh-trusted-ca-bundle-configmap.yaml" + odhKserveCustomCABundleConfigMapPath = "./testdata/configmaps/odh-kserve-custom-ca-cert-configmap.yaml" +) + +func TestControllers(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Controller Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{ + filepath.Join("..", "..", "..", "config", "crd", "bases"), + filepath.Join("..", "..", "..", "config", "crd", "external"), + }, + ErrorIfCRDPathMissing: true, + + // The BinaryAssetsDirectory is only required if you want to run the tests directly + // without call the makefile target test. If not informed it will look for the + // default path defined in controller-runtime which is /usr/local/kubebuilder/. + // Note that you must have the required binaries setup under the bin directory to perform + // the tests directly. When we run make test it will be setup and used automatically. + BinaryAssetsDirectory: filepath.Join("..", "..", "..", "bin", "k8s", + fmt.Sprintf("1.31.0-%s-%s", runtime.GOOS, runtime.GOARCH)), + } + + var err error + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + // err = corev1.AddToScheme(scheme.Scheme) + utils.RegisterSchemes(scheme.Scheme) + // Expect(err).NotTo(HaveOccurred()) + + // +kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme.Scheme, + Client: client.Options{ + Cache: &client.CacheOptions{ + DisableFor: []client.Object{&istiov1beta1.AuthorizationPolicy{}}, + }, + }, + Cache: cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + &corev1.Secret{}: { + Label: labels.SelectorFromSet(labels.Set{ + "opendatahub.io/managed": "true", + }), + }, + }, + }, + Metrics: metricsserver.Options{BindAddress: "0"}, + }) + Expect(err).NotTo(HaveOccurred()) + + err = (&SecretReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&ConfigMapReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + + go func() { + defer GinkgoRecover() + err = mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred(), "failed to start manager") + }() +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + cancel() + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) + +// Cleanup resources to not contaminate between tests +var _ = AfterEach(func() { + cleanUp := func(namespace string, cli client.Client) { + inNamespace := client.InNamespace(namespace) + _, meshNamespace := utils.GetIstioControlPlaneName(ctx, cli) + istioNamespace := client.InNamespace(meshNamespace) + Expect(cli.DeleteAllOf(context.TODO(), &kservev1alpha1.ServingRuntime{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &kservev1beta1.InferenceService{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &routev1.Route{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &monitoringv1.ServiceMonitor{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &k8srbacv1.RoleBinding{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &corev1.Secret{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &authorinov1beta2.AuthConfig{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &corev1.ConfigMap{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &corev1.Service{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &istioclientv1beta1.Gateway{}, istioNamespace)).ToNot(HaveOccurred()) + } + cleanUp(WorkingNamespace, k8sClient) + for _, ns := range testutils.Namespaces.All() { + cleanUp(ns, k8sClient) + } + testutils.Namespaces.Clear() +}) + +func convertToStructuredResource(path string, out k8sRuntime.Object) error { + data, err := os.ReadFile(path) + if err != nil { + return err + } + + return utils.ConvertToStructuredResource(data, out) +} + +func waitForConfigMap(cli client.Client, namespace, configMapName string, maxTries int, delay time.Duration) (*corev1.ConfigMap, error) { + time.Sleep(delay) + + ctx := context.Background() + configMap := &corev1.ConfigMap{} + for try := 1; try <= maxTries; try++ { + err := cli.Get(ctx, client.ObjectKey{Namespace: namespace, Name: configMapName}, configMap) + if err == nil { + return configMap, nil + } + if !apierrs.IsNotFound(err) { + return nil, fmt.Errorf("failed to get configmap %s/%s: %v", namespace, configMapName, err) + } + + if try > maxTries { + time.Sleep(1 * time.Second) + return nil, err + } + } + return configMap, nil +} diff --git a/controllers/testdata/configmaps/odh-kserve-custom-ca-cert-configmap-updated.yaml b/internal/controller/core/testdata/configmaps/odh-kserve-custom-ca-cert-configmap-updated.yaml similarity index 100% rename from controllers/testdata/configmaps/odh-kserve-custom-ca-cert-configmap-updated.yaml rename to internal/controller/core/testdata/configmaps/odh-kserve-custom-ca-cert-configmap-updated.yaml diff --git a/controllers/testdata/configmaps/odh-kserve-custom-ca-cert-configmap.yaml b/internal/controller/core/testdata/configmaps/odh-kserve-custom-ca-cert-configmap.yaml similarity index 100% rename from controllers/testdata/configmaps/odh-kserve-custom-ca-cert-configmap.yaml rename to internal/controller/core/testdata/configmaps/odh-kserve-custom-ca-cert-configmap.yaml diff --git a/controllers/testdata/configmaps/odh-trusted-ca-bundle-configmap-updated.yaml b/internal/controller/core/testdata/configmaps/odh-trusted-ca-bundle-configmap-updated.yaml similarity index 100% rename from controllers/testdata/configmaps/odh-trusted-ca-bundle-configmap-updated.yaml rename to internal/controller/core/testdata/configmaps/odh-trusted-ca-bundle-configmap-updated.yaml diff --git a/controllers/testdata/configmaps/odh-trusted-ca-bundle-configmap.yaml b/internal/controller/core/testdata/configmaps/odh-trusted-ca-bundle-configmap.yaml similarity index 100% rename from controllers/testdata/configmaps/odh-trusted-ca-bundle-configmap.yaml rename to internal/controller/core/testdata/configmaps/odh-trusted-ca-bundle-configmap.yaml diff --git a/controllers/testdata/secrets/dataconnection-string.yaml b/internal/controller/core/testdata/secrets/dataconnection-string.yaml similarity index 100% rename from controllers/testdata/secrets/dataconnection-string.yaml rename to internal/controller/core/testdata/secrets/dataconnection-string.yaml diff --git a/controllers/testdata/secrets/storageconfig-cert-encoded.yaml b/internal/controller/core/testdata/secrets/storageconfig-cert-encoded.yaml similarity index 100% rename from controllers/testdata/secrets/storageconfig-cert-encoded.yaml rename to internal/controller/core/testdata/secrets/storageconfig-cert-encoded.yaml diff --git a/controllers/testdata/secrets/storageconfig-cert-string.yaml b/internal/controller/core/testdata/secrets/storageconfig-cert-string.yaml similarity index 100% rename from controllers/testdata/secrets/storageconfig-cert-string.yaml rename to internal/controller/core/testdata/secrets/storageconfig-cert-string.yaml diff --git a/controllers/testdata/secrets/storageconfig-encoded-unmanaged.yaml b/internal/controller/core/testdata/secrets/storageconfig-encoded-unmanaged.yaml similarity index 100% rename from controllers/testdata/secrets/storageconfig-encoded-unmanaged.yaml rename to internal/controller/core/testdata/secrets/storageconfig-encoded-unmanaged.yaml diff --git a/controllers/testdata/secrets/storageconfig-encoded.yaml b/internal/controller/core/testdata/secrets/storageconfig-encoded.yaml similarity index 100% rename from controllers/testdata/secrets/storageconfig-encoded.yaml rename to internal/controller/core/testdata/secrets/storageconfig-encoded.yaml diff --git a/controllers/testdata/secrets/storageconfig-updated-cert-encoded.yaml b/internal/controller/core/testdata/secrets/storageconfig-updated-cert-encoded.yaml similarity index 100% rename from controllers/testdata/secrets/storageconfig-updated-cert-encoded.yaml rename to internal/controller/core/testdata/secrets/storageconfig-updated-cert-encoded.yaml diff --git a/controllers/nim_account_controller.go b/internal/controller/nim/account_controller.go similarity index 85% rename from controllers/nim_account_controller.go rename to internal/controller/nim/account_controller.go index ffd074d0..e3048a05 100644 --- a/controllers/nim_account_controller.go +++ b/internal/controller/nim/account_controller.go @@ -1,4 +1,5 @@ /* +Copyright 2024. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -13,17 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controllers +package nim import ( "context" "encoding/json" "fmt" - "github.com/go-logr/logr" - "github.com/kuadrant/authorino/pkg/log" - "github.com/opendatahub-io/odh-model-controller/api/nim/v1" - "github.com/opendatahub-io/odh-model-controller/controllers/constants" - "github.com/opendatahub-io/odh-model-controller/controllers/utils" + "strings" + templatev1 "github.com/openshift/api/template/v1" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -40,18 +38,21 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "strings" -) -type ( - NimAccountReconciler struct { - client.Client - Log logr.Logger - KClient kubernetes.Interface - } + v1 "github.com/opendatahub-io/odh-model-controller/api/nim/v1" + "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" ) +// AccountReconciler reconciles a Account object +type AccountReconciler struct { + client.Client + Scheme *runtime.Scheme + KClient kubernetes.Interface +} + const ( apiKeySpecPath = "spec.apiKeySecret.name" ) @@ -60,13 +61,22 @@ var ( labels = map[string]string{"opendatahub.io/managed": "true"} ) -func (r *NimAccountReconciler) SetupWithManager(mgr ctrl.Manager, ctx context.Context) error { +// +kubebuilder:rbac:groups=nim.opendatahub.io,resources=accounts,verbs=get;list;watch;update +// +kubebuilder:rbac:groups=nim.opendatahub.io,resources=accounts/status,verbs=get;list;watch;update +// +kubebuilder:rbac:groups=nim.opendatahub.io,resources=accounts/finalizers,verbs=update +// +kubebuilder:rbac:groups=template.openshift.io,resources=templates,verbs=get;list;watch;create;update;delete + +func (r *AccountReconciler) SetupWithManager(mgr ctrl.Manager, ctx context.Context) error { + // TODO: Copied from original main.go... Should it be FromContext? + logger := ctrl.Log.WithName("controllers").WithName("ModelRegistryInferenceService") + if err := mgr.GetFieldIndexer().IndexField(ctx, &v1.Account{}, apiKeySpecPath, func(obj client.Object) []string { return []string{obj.(*v1.Account).Spec.APIKeySecret.Name} }); err != nil { - r.Log.Error(err, "failed to set cache index") + logger.Error(err, "failed to set cache index") return err } + return ctrl.NewControllerManagedBy(mgr). Named("odh-nim-controller"). For(&v1.Account{}). @@ -78,7 +88,7 @@ func (r *NimAccountReconciler) SetupWithManager(mgr ctrl.Manager, ctx context.Co var requests []reconcile.Request accounts := &v1.AccountList{} if err := mgr.GetClient().List(ctx, accounts, client.MatchingFields{apiKeySpecPath: obj.GetName()}); err != nil { - r.Log.Error(err, "failed to fetch accounts") + logger.Error(err, "failed to fetch accounts") return requests } for _, item := range accounts.Items { @@ -92,8 +102,17 @@ func (r *NimAccountReconciler) SetupWithManager(mgr ctrl.Manager, ctx context.Co Complete(r) } -func (r *NimAccountReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - logger := r.Log.WithValues("Account", req.Name, "namespace", req.Namespace) +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the Account object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/reconcile +func (r *AccountReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx).WithValues("Account", req.Name, "namespace", req.Namespace) ctx = log.IntoContext(ctx, logger) account := &v1.Account{} @@ -204,7 +223,7 @@ func (r *NimAccountReconciler) Reconcile(ctx context.Context, req ctrl.Request) meta.SetStatusCondition(&targetStatus.Conditions, makeConfigMapFailureCondition(account.Generation, msg)) return ctrl.Result{}, err } else { - ref, refErr := reference.GetReference(r.Scheme(), cm) + ref, refErr := reference.GetReference(r.Scheme, cm) if refErr != nil { return ctrl.Result{}, refErr } @@ -223,7 +242,7 @@ func (r *NimAccountReconciler) Reconcile(ctx context.Context, req ctrl.Request) meta.SetStatusCondition(&targetStatus.Conditions, makeTemplateFailureCondition(account.Generation, msg)) return ctrl.Result{}, err } else { - ref, refErr := reference.GetReference(r.Scheme(), template) + ref, refErr := reference.GetReference(r.Scheme, template) if refErr != nil { return ctrl.Result{}, refErr } @@ -242,7 +261,7 @@ func (r *NimAccountReconciler) Reconcile(ctx context.Context, req ctrl.Request) meta.SetStatusCondition(&targetStatus.Conditions, makePullSecretFailureCondition(account.Generation, msg)) return ctrl.Result{}, err } else { - ref, refErr := reference.GetReference(r.Scheme(), pullSecret) + ref, refErr := reference.GetReference(r.Scheme, pullSecret) if refErr != nil { return ctrl.Result{}, refErr } @@ -257,7 +276,7 @@ func (r *NimAccountReconciler) Reconcile(ctx context.Context, req ctrl.Request) } // reconcileNimConfig is used for reconciling the configmap encapsulating the model data used for constructing inference services -func (r *NimAccountReconciler) reconcileNimConfig( +func (r *AccountReconciler) reconcileNimConfig( ctx context.Context, ownerCfg *ssametav1.OwnerReferenceApplyConfiguration, namespace, apiKey string, runtimes []utils.NimRuntime, ) (*corev1.ConfigMap, error) { @@ -281,7 +300,7 @@ func (r *NimAccountReconciler) reconcileNimConfig( } // reconcileRuntimeTemplate is used for reconciling the template encapsulating the serving runtime -func (r *NimAccountReconciler) reconcileRuntimeTemplate(ctx context.Context, account *v1.Account) (*templatev1.Template, error) { +func (r *AccountReconciler) reconcileRuntimeTemplate(ctx context.Context, account *v1.Account) (*templatev1.Template, error) { template := &templatev1.Template{ ObjectMeta: metav1.ObjectMeta{ Name: constants.NimRuntimeTemplateName, @@ -290,7 +309,7 @@ func (r *NimAccountReconciler) reconcileRuntimeTemplate(ctx context.Context, acc } if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, template, func() error { - if err := controllerutil.SetControllerReference(account, template, r.Scheme()); err != nil { + if err := controllerutil.SetControllerReference(account, template, r.Scheme); err != nil { return err } @@ -301,7 +320,7 @@ func (r *NimAccountReconciler) reconcileRuntimeTemplate(ctx context.Context, acc template.Labels = labels - sr, srErr := utils.GetNimServingRuntimeTemplate(r.Scheme()) + sr, srErr := utils.GetNimServingRuntimeTemplate(r.Scheme) if srErr != nil { return srErr } @@ -318,7 +337,7 @@ func (r *NimAccountReconciler) reconcileRuntimeTemplate(ctx context.Context, acc } // reconcileNimPullSecret is used to reconcile the pull secret for pulling the custom runtime images -func (r *NimAccountReconciler) reconcileNimPullSecret( +func (r *AccountReconciler) reconcileNimPullSecret( ctx context.Context, ownerCfg *ssametav1.OwnerReferenceApplyConfiguration, namespace, apiKey string, ) (*corev1.Secret, error) { creds := map[string]map[string]map[string]string{ @@ -351,7 +370,7 @@ func (r *NimAccountReconciler) reconcileNimPullSecret( } // updateStatus is used for fetching an updating the status of the account -func (r *NimAccountReconciler) updateStatus(ctx context.Context, subject types.NamespacedName, status v1.AccountStatus) { +func (r *AccountReconciler) updateStatus(ctx context.Context, subject types.NamespacedName, status v1.AccountStatus) { logger := log.FromContext(ctx) logger.V(1).Info("updating status") @@ -369,11 +388,11 @@ func (r *NimAccountReconciler) updateStatus(ctx context.Context, subject types.N } // createOwnerReferenceCfg is used to create an owner reference config to use with server side apply -func (r *NimAccountReconciler) createOwnerReferenceCfg(account *v1.Account) *ssametav1.OwnerReferenceApplyConfiguration { +func (r *AccountReconciler) createOwnerReferenceCfg(account *v1.Account) *ssametav1.OwnerReferenceApplyConfiguration { // we fetch the gvk instead of getting the kind and apiversion from the object, because of an alleged envtest bug // stripping down all objects typemeta. This is the PR comment discussing this: // https://github.com/opendatahub-io/odh-model-controller/pull/289#discussion_r1833811970 - gvk, _ := apiutil.GVKForObject(account, r.Scheme()) + gvk, _ := apiutil.GVKForObject(account, r.Scheme) return ssametav1.OwnerReference(). WithKind(gvk.Kind). WithName(account.Name). @@ -384,7 +403,7 @@ func (r *NimAccountReconciler) createOwnerReferenceCfg(account *v1.Account) *ssa } // cleanupResources is used for deleting the integration related resources (configmap, template, pull secret) -func (r *NimAccountReconciler) cleanupResources(ctx context.Context, account *v1.Account) { +func (r *AccountReconciler) cleanupResources(ctx context.Context, account *v1.Account) { logger := log.FromContext(ctx) logger.V(1).Info("cleaning up") diff --git a/controllers/nim_account_controller_test.go b/internal/controller/nim/account_controller_test.go similarity index 73% rename from controllers/nim_account_controller_test.go rename to internal/controller/nim/account_controller_test.go index 6d8dd167..88bf8c1c 100644 --- a/controllers/nim_account_controller_test.go +++ b/internal/controller/nim/account_controller_test.go @@ -1,4 +1,5 @@ /* +Copyright 2024. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -13,17 +14,15 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controllers +package nim import ( "context" "errors" "fmt" - . "github.com/onsi/ginkgo" + + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - v1 "github.com/opendatahub-io/odh-model-controller/api/nim/v1" - "github.com/opendatahub-io/odh-model-controller/controllers/testdata" - "github.com/opendatahub-io/odh-model-controller/controllers/utils" templatev1 "github.com/openshift/api/template/v1" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -33,6 +32,10 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/reference" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + + v1 "github.com/opendatahub-io/odh-model-controller/api/nim/v1" + "github.com/opendatahub-io/odh-model-controller/internal/controller/testdata" + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" ) var _ = Describe("NIM Account Controller Test Cases", func() { @@ -46,7 +49,7 @@ var _ = Describe("NIM Account Controller Test Cases", func() { By("Create testing Namespace " + nameNs) testNs := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: nameNs}} - Expect(cli.Create(ctx, testNs)).To(Succeed()) + Expect(k8sClient.Create(ctx, testNs)).To(Succeed()) By("Create an Account and an API Key Secret") acctSubject := types.NamespacedName{Name: nameNs, Namespace: nameNs} @@ -57,21 +60,21 @@ var _ = Describe("NIM Account Controller Test Cases", func() { assertSuccessfulAccount(acctSubject, account).Should(Succeed()) By("Verify resources created") - expectedOwner := createOwnerReference(cli.Scheme(), account) + expectedOwner := createOwnerReference(k8sClient.Scheme(), account) dataCmap := &corev1.ConfigMap{} dataCmapSubject := namespacedNameFromReference(account.Status.NIMConfig) - Expect(cli.Get(ctx, dataCmapSubject, dataCmap)).To(Succeed()) + Expect(k8sClient.Get(ctx, dataCmapSubject, dataCmap)).To(Succeed()) Expect(dataCmap.OwnerReferences[0]).To(Equal(expectedOwner)) runtimeTemplate := &templatev1.Template{} runtimeTemplateSubject := namespacedNameFromReference(account.Status.RuntimeTemplate) - Expect(cli.Get(ctx, runtimeTemplateSubject, runtimeTemplate)).To(Succeed()) + Expect(k8sClient.Get(ctx, runtimeTemplateSubject, runtimeTemplate)).To(Succeed()) Expect(runtimeTemplate.OwnerReferences[0]).To(Equal(expectedOwner)) pullSecret := &corev1.Secret{} pullSecretSubject := namespacedNameFromReference(account.Status.NIMPullSecret) - Expect(cli.Get(ctx, pullSecretSubject, pullSecret)).To(Succeed()) + Expect(k8sClient.Get(ctx, pullSecretSubject, pullSecret)).To(Succeed()) Expect(pullSecret.OwnerReferences[0]).To(Equal(expectedOwner)) By("Verify models info") @@ -80,17 +83,17 @@ var _ = Describe("NIM Account Controller Test Cases", func() { By("Cleanups") apiKeySecret := &corev1.Secret{} apiKeySubject := namespacedNameFromReference(&account.Spec.APIKeySecret) - Expect(cli.Get(ctx, apiKeySubject, apiKeySecret)).Should(Succeed()) + Expect(k8sClient.Get(ctx, apiKeySubject, apiKeySecret)).Should(Succeed()) - Expect(cli.Delete(ctx, account)).To(Succeed()) - Expect(cli.Delete(ctx, apiKeySecret)).To(Succeed()) + Expect(k8sClient.Delete(ctx, account)).To(Succeed()) + Expect(k8sClient.Delete(ctx, apiKeySecret)).To(Succeed()) // we delete the following because K8S GC is not working in envtest - Expect(cli.Delete(ctx, dataCmap)).To(Succeed()) - Expect(cli.Delete(ctx, runtimeTemplate)).To(Succeed()) - Expect(cli.Delete(ctx, pullSecret)).To(Succeed()) + Expect(k8sClient.Delete(ctx, dataCmap)).To(Succeed()) + Expect(k8sClient.Delete(ctx, runtimeTemplate)).To(Succeed()) + Expect(k8sClient.Delete(ctx, pullSecret)).To(Succeed()) - Expect(cli.Delete(ctx, testNs)).To(Succeed()) + Expect(k8sClient.Delete(ctx, testNs)).To(Succeed()) }) It("Should not reconcile resources for an account with an invalid API key", func() { @@ -98,7 +101,7 @@ var _ = Describe("NIM Account Controller Test Cases", func() { By("Create testing Namespace " + nameNs) testNs := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: nameNs}} - Expect(cli.Create(ctx, testNs)).To(Succeed()) + Expect(k8sClient.Create(ctx, testNs)).To(Succeed()) By("Create an Account and a wrong API Key Secret") acctSubject := types.NamespacedName{Name: nameNs, Namespace: nameNs} @@ -111,11 +114,11 @@ var _ = Describe("NIM Account Controller Test Cases", func() { By("Cleanups") apiKeySecret := &corev1.Secret{} apiKeySubject := namespacedNameFromReference(&account.Spec.APIKeySecret) - Expect(cli.Get(ctx, apiKeySubject, apiKeySecret)).Should(Succeed()) + Expect(k8sClient.Get(ctx, apiKeySubject, apiKeySecret)).Should(Succeed()) - Expect(cli.Delete(ctx, apiKeySecret)).Should(Succeed()) - Expect(cli.Delete(ctx, account)).Should(Succeed()) - Expect(cli.Delete(ctx, testNs)).To(Succeed()) + Expect(k8sClient.Delete(ctx, apiKeySecret)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, account)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, testNs)).To(Succeed()) }) It("Should remove all resources if the API key Secret was deleted", func() { @@ -124,7 +127,7 @@ var _ = Describe("NIM Account Controller Test Cases", func() { By("Create testing Namespace " + nameNs) testNs := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: nameNs}} - Expect(cli.Create(ctx, testNs)).To(Succeed()) + Expect(k8sClient.Create(ctx, testNs)).To(Succeed()) By("Create an Account and an API Key Secret") acctSubject := types.NamespacedName{Name: nameNs, Namespace: nameNs} @@ -136,37 +139,37 @@ var _ = Describe("NIM Account Controller Test Cases", func() { By("Verify resources created") dataCmapSubject := namespacedNameFromReference(account.Status.NIMConfig) - Expect(cli.Get(ctx, dataCmapSubject, &corev1.ConfigMap{})).To(Succeed()) + Expect(k8sClient.Get(ctx, dataCmapSubject, &corev1.ConfigMap{})).To(Succeed()) runtimeTemplateSubject := namespacedNameFromReference(account.Status.RuntimeTemplate) - Expect(cli.Get(ctx, runtimeTemplateSubject, &templatev1.Template{})).To(Succeed()) + Expect(k8sClient.Get(ctx, runtimeTemplateSubject, &templatev1.Template{})).To(Succeed()) pullSecretSubject := namespacedNameFromReference(account.Status.NIMPullSecret) - Expect(cli.Get(ctx, pullSecretSubject, &corev1.Secret{})).To(Succeed()) + Expect(k8sClient.Get(ctx, pullSecretSubject, &corev1.Secret{})).To(Succeed()) By("Delete API key Secret") apiKeySecret := &corev1.Secret{} apiKeySubject := namespacedNameFromReference(&account.Spec.APIKeySecret) - Expect(cli.Get(ctx, apiKeySubject, apiKeySecret)).Should(Succeed()) - Expect(cli.Delete(ctx, apiKeySecret)).To(Succeed()) + Expect(k8sClient.Get(ctx, apiKeySubject, apiKeySecret)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, apiKeySecret)).To(Succeed()) By("Verify resources deleted") Eventually(func() error { - if err := cli.Get(ctx, dataCmapSubject, &corev1.ConfigMap{}); !k8serrors.IsNotFound(err) { + if err := k8sClient.Get(ctx, dataCmapSubject, &corev1.ConfigMap{}); !k8serrors.IsNotFound(err) { return fmt.Errorf("expected configmap to be deleted") } - if err := cli.Get(ctx, runtimeTemplateSubject, &templatev1.Template{}); !k8serrors.IsNotFound(err) { + if err := k8sClient.Get(ctx, runtimeTemplateSubject, &templatev1.Template{}); !k8serrors.IsNotFound(err) { return fmt.Errorf("expected template to be deleted") } - if err := cli.Get(ctx, pullSecretSubject, &corev1.Secret{}); !k8serrors.IsNotFound(err) { + if err := k8sClient.Get(ctx, pullSecretSubject, &corev1.Secret{}); !k8serrors.IsNotFound(err) { return fmt.Errorf("expected pull secret to be deleted") } return nil - }, timeout, interval).Should(Succeed()) + }, testTimeout, testInterval).Should(Succeed()) By("Cleanups") - Expect(cli.Delete(ctx, account)).To(Succeed()) - Expect(cli.Delete(ctx, testNs)).To(Succeed()) + Expect(k8sClient.Delete(ctx, account)).To(Succeed()) + Expect(k8sClient.Delete(ctx, testNs)).To(Succeed()) }) }) @@ -175,14 +178,15 @@ func createApiKeySecretAndAccount(account types.NamespacedName, apiKey string) { ObjectMeta: metav1.ObjectMeta{ Name: account.Name + "-api-key", Namespace: account.Namespace, + Labels: map[string]string{"opendatahub.io/managed": "true"}, }, Data: map[string][]byte{ "api_key": []byte(apiKey), }, } - Expect(cli.Create(ctx, apiKeySecret)).To(Succeed()) + Expect(k8sClient.Create(ctx, apiKeySecret)).To(Succeed()) - apiKeyRef, _ := reference.GetReference(cli.Scheme(), apiKeySecret) + apiKeyRef, _ := reference.GetReference(k8sClient.Scheme(), apiKeySecret) acct := &v1.Account{ ObjectMeta: metav1.ObjectMeta{ Name: account.Name, @@ -192,12 +196,12 @@ func createApiKeySecretAndAccount(account types.NamespacedName, apiKey string) { APIKeySecret: *apiKeyRef, }, } - Expect(cli.Create(ctx, acct)).To(Succeed()) + Expect(k8sClient.Create(ctx, acct)).To(Succeed()) } func assertSuccessfulAccount(acctSubject types.NamespacedName, account *v1.Account) AsyncAssertion { return Eventually(func() error { - if err := cli.Get(ctx, acctSubject, account); err != nil { + if err := k8sClient.Get(ctx, acctSubject, account); err != nil { return err } @@ -214,12 +218,12 @@ func assertSuccessfulAccount(acctSubject types.NamespacedName, account *v1.Accou } } return nil - }, timeout, interval) + }, testTimeout, testInterval) } func assertFailedAccount(acctSubject types.NamespacedName, account *v1.Account) AsyncAssertion { return Eventually(func() error { - if err := cli.Get(ctx, acctSubject, account); err != nil { + if err := k8sClient.Get(ctx, acctSubject, account); err != nil { return err } @@ -254,7 +258,7 @@ func assertFailedAccount(acctSubject types.NamespacedName, account *v1.Account) } } return nil - }, timeout, interval) + }, testTimeout, testInterval) } func createOwnerReference(scheme *runtime.Scheme, account *v1.Account) metav1.OwnerReference { diff --git a/internal/controller/nim/suite_test.go b/internal/controller/nim/suite_test.go new file mode 100644 index 00000000..74f95182 --- /dev/null +++ b/internal/controller/nim/suite_test.go @@ -0,0 +1,176 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nim + +import ( + "context" + "fmt" + "path/filepath" + "runtime" + "testing" + "time" + + kservev1alpha1 "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" + kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" + authorinov1beta2 "github.com/kuadrant/authorino/api/v1beta2" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + routev1 "github.com/openshift/api/route/v1" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + istioclientv1beta1 "istio.io/client-go/pkg/apis/networking/v1beta1" + istiov1beta1 "istio.io/client-go/pkg/apis/security/v1beta1" + corev1 "k8s.io/api/core/v1" + k8srbacv1 "k8s.io/api/rbac/v1" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + k8sLabels "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" + testutils "github.com/opendatahub-io/odh-model-controller/test/utils" + // +kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var cfg *rest.Config +var k8sClient client.Client +var testEnv *envtest.Environment +var ctx context.Context +var cancel context.CancelFunc + +// TODO: Deduplicate +const testTimeout = time.Second * 20 +const testInterval = time.Millisecond * 10 +const WorkingNamespace = "default" + +func TestControllers(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Controller & Webhook Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{ + filepath.Join("..", "..", "..", "config", "crd", "bases"), + filepath.Join("..", "..", "..", "config", "crd", "external"), + }, + ErrorIfCRDPathMissing: true, + + // The BinaryAssetsDirectory is only required if you want to run the tests directly + // without call the makefile target test. If not informed it will look for the + // default path defined in controller-runtime which is /usr/local/kubebuilder/. + // Note that you must have the required binaries setup under the bin directory to perform + // the tests directly. When we run make test it will be setup and used automatically. + BinaryAssetsDirectory: filepath.Join("..", "..", "bin", "k8s", + fmt.Sprintf("1.31.0-%s-%s", runtime.GOOS, runtime.GOARCH)), + } + + var err error + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + utils.RegisterSchemes(scheme.Scheme) + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme.Scheme, + Client: client.Options{ + Cache: &client.CacheOptions{ + DisableFor: []client.Object{&istiov1beta1.AuthorizationPolicy{}}, + }, + }, + Cache: cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + &corev1.Secret{}: { + Label: k8sLabels.SelectorFromSet(k8sLabels.Set{ + "opendatahub.io/managed": "true", + }), + }, + }, + }, + Metrics: metricsserver.Options{BindAddress: "0"}, + }) + Expect(err).NotTo(HaveOccurred()) + + kubeClient, kubeClientErr := kubernetes.NewForConfig(cfg) + Expect(kubeClientErr).NotTo(HaveOccurred()) + + err = (&AccountReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + KClient: kubeClient, + }).SetupWithManager(mgr, ctrl.SetupSignalHandler()) + Expect(err).NotTo(HaveOccurred()) + + go func() { + defer GinkgoRecover() + err = mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred(), "failed to start manager") + }() +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + cancel() + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) + +// Cleanup resources to not contaminate between tests +var _ = AfterEach(func() { + cleanUp := func(namespace string, cli client.Client) { + inNamespace := client.InNamespace(namespace) + _, meshNamespace := utils.GetIstioControlPlaneName(ctx, cli) + istioNamespace := client.InNamespace(meshNamespace) + Expect(cli.DeleteAllOf(context.TODO(), &kservev1alpha1.ServingRuntime{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &kservev1beta1.InferenceService{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &routev1.Route{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &monitoringv1.ServiceMonitor{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &k8srbacv1.RoleBinding{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &corev1.Secret{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &authorinov1beta2.AuthConfig{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &corev1.ConfigMap{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &corev1.Service{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &istioclientv1beta1.Gateway{}, istioNamespace)).ToNot(HaveOccurred()) + } + cleanUp(WorkingNamespace, k8sClient) + for _, ns := range testutils.Namespaces.All() { + cleanUp(ns, k8sClient) + } + testutils.Namespaces.Clear() +}) diff --git a/controllers/testdata/nim/ngc_catalog_response_page_0.json b/internal/controller/nim/testdata/ngc_catalog_response_page_0.json similarity index 100% rename from controllers/testdata/nim/ngc_catalog_response_page_0.json rename to internal/controller/nim/testdata/ngc_catalog_response_page_0.json diff --git a/controllers/testdata/nim/ngc_catalog_response_page_1.json b/internal/controller/nim/testdata/ngc_catalog_response_page_1.json similarity index 100% rename from controllers/testdata/nim/ngc_catalog_response_page_1.json rename to internal/controller/nim/testdata/ngc_catalog_response_page_1.json diff --git a/controllers/testdata/nim/ngc_model_llama-3_1-8b-instruct_response.json b/internal/controller/nim/testdata/ngc_model_llama-3_1-8b-instruct_response.json similarity index 100% rename from controllers/testdata/nim/ngc_model_llama-3_1-8b-instruct_response.json rename to internal/controller/nim/testdata/ngc_model_llama-3_1-8b-instruct_response.json diff --git a/controllers/testdata/nim/ngc_model_phi-3-mini-4k-instruct_response.json b/internal/controller/nim/testdata/ngc_model_phi-3-mini-4k-instruct_response.json similarity index 100% rename from controllers/testdata/nim/ngc_model_phi-3-mini-4k-instruct_response.json rename to internal/controller/nim/testdata/ngc_model_phi-3-mini-4k-instruct_response.json diff --git a/controllers/testdata/nim/ngc_token_response.json b/internal/controller/nim/testdata/ngc_token_response.json similarity index 100% rename from controllers/testdata/nim/ngc_token_response.json rename to internal/controller/nim/testdata/ngc_token_response.json diff --git a/controllers/testdata/nim/runtime_token_response.json b/internal/controller/nim/testdata/runtime_token_response.json similarity index 100% rename from controllers/testdata/nim/runtime_token_response.json rename to internal/controller/nim/testdata/runtime_token_response.json diff --git a/controllers/processors/deltaProcessor.go b/internal/controller/processors/deltaProcessor.go similarity index 93% rename from controllers/processors/deltaProcessor.go rename to internal/controller/processors/deltaProcessor.go index c50ba743..2f93117b 100644 --- a/controllers/processors/deltaProcessor.go +++ b/internal/controller/processors/deltaProcessor.go @@ -16,8 +16,8 @@ limitations under the License. package processors import ( - "github.com/opendatahub-io/odh-model-controller/controllers/comparators" - "github.com/opendatahub-io/odh-model-controller/controllers/utils" + "github.com/opendatahub-io/odh-model-controller/internal/controller/comparators" + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/controllers/resources/authconfig.go b/internal/controller/resources/authconfig.go similarity index 96% rename from controllers/resources/authconfig.go rename to internal/controller/resources/authconfig.go index 71d1f9d8..28d38917 100644 --- a/controllers/resources/authconfig.go +++ b/internal/controller/resources/authconfig.go @@ -27,11 +27,12 @@ import ( kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" authorinov1beta2 "github.com/kuadrant/authorino/api/v1beta2" - "github.com/opendatahub-io/odh-model-controller/controllers/constants" - "github.com/opendatahub-io/odh-model-controller/controllers/utils" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" ) type AuthType string @@ -96,7 +97,7 @@ func (s *staticTemplateLoader) Load(ctx context.Context, authType AuthType, key resolvedTemplate, err := s.resolveTemplate(template, templateData) if err != nil { - return authConfig, fmt.Errorf("could not resovle auth template. cause %w", err) + return authConfig, fmt.Errorf("could not resolve auth template. cause %w", err) } err = utils.ConvertToStructuredResource(resolvedTemplate, &authConfig) if err != nil { @@ -132,7 +133,7 @@ func NewConfigMapTemplateLoader(client client.Client, fallback AuthConfigTemplat } func (c *configMapTemplateLoader) Load(ctx context.Context, authType AuthType, key types.NamespacedName) (authorinov1beta2.AuthConfig, error) { - // TOOD: check "authconfig-template" CM in key.Namespace to see if there is a "spec" to use, construct a AuthConfig object + // TODO: check "authconfig-template" CM in key.Namespace to see if there is a "spec" to use, construct a AuthConfig object // https://issues.redhat.com/browse/RHOAIENG-847 // else diff --git a/controllers/resources/authconfig_test.go b/internal/controller/resources/authconfig_test.go similarity index 92% rename from controllers/resources/authconfig_test.go rename to internal/controller/resources/authconfig_test.go index 9f543de7..67fd39aa 100644 --- a/controllers/resources/authconfig_test.go +++ b/internal/controller/resources/authconfig_test.go @@ -7,13 +7,14 @@ import ( kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/opendatahub-io/odh-model-controller/controllers/resources" "github.com/tidwall/gjson" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "knative.dev/pkg/apis" duckv1 "knative.dev/pkg/apis/duck/v1" knservingv1 "knative.dev/serving/pkg/apis/serving/v1" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/resources" ) var _ = When("InferenceService is ready", func() { @@ -79,7 +80,7 @@ var _ = When("InferenceService is ready", func() { Context("Template loading", func() { - It("should resovle UserDefined template", func() { + It("should resolve UserDefined template", func() { typeName := types.NamespacedName{ Name: "test", Namespace: "test-ns", @@ -115,8 +116,10 @@ var _ = When("InferenceService is ready", func() { Namespace: "test-ns", } - os.Setenv("AUTH_AUDIENCE", "http://test.com") - defer os.Unsetenv("AUTH_AUDIENCE") + _ = os.Setenv("AUTH_AUDIENCE", "http://test.com") + defer func() { + _ = os.Unsetenv("AUTH_AUDIENCE") + }() ac, err := resources.NewStaticTemplateLoader().Load( context.Background(), @@ -148,8 +151,10 @@ var _ = When("InferenceService is ready", func() { Namespace: "test-ns", } - os.Setenv("AUTHORINO_LABEL", "opendatahub=test") - defer os.Unsetenv("AUTHORINO_LABEL") + _ = os.Setenv("AUTHORINO_LABEL", "opendatahub=test") + defer func() { + _ = os.Unsetenv("AUTHORINO_LABEL") + }() ac, err := resources.NewStaticTemplateLoader().Load( context.Background(), diff --git a/controllers/resources/clusterrolebinding.go b/internal/controller/resources/clusterrolebinding.go similarity index 95% rename from controllers/resources/clusterrolebinding.go rename to internal/controller/resources/clusterrolebinding.go index bdccfa24..fd54e9fd 100644 --- a/controllers/resources/clusterrolebinding.go +++ b/internal/controller/resources/clusterrolebinding.go @@ -20,13 +20,14 @@ import ( "fmt" "github.com/go-logr/logr" - "github.com/opendatahub-io/odh-model-controller/controllers/comparators" - "github.com/opendatahub-io/odh-model-controller/controllers/processors" v1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/comparators" + "github.com/opendatahub-io/odh-model-controller/internal/controller/processors" ) type ClusterRoleBindingHandler interface { @@ -122,13 +123,13 @@ func (r *clusterRoleBindingHandler) ProcessDelta(ctx context.Context, log logr.L rp.Subjects = desiredCRB.Subjects if err = r.client.Update(ctx, rp); err != nil { - return + return err } } if delta.IsRemoved() { log.V(1).Info("Delta found", "delete", existingCRB.GetName()) if err = r.client.Delete(ctx, existingCRB); err != nil { - return + return err } } return nil diff --git a/controllers/resources/configmap.go b/internal/controller/resources/configmap.go similarity index 100% rename from controllers/resources/configmap.go rename to internal/controller/resources/configmap.go diff --git a/controllers/resources/gateway.go b/internal/controller/resources/gateway.go similarity index 100% rename from controllers/resources/gateway.go rename to internal/controller/resources/gateway.go diff --git a/controllers/resources/networkpolicy.go b/internal/controller/resources/networkpolicy.go similarity index 98% rename from controllers/resources/networkpolicy.go rename to internal/controller/resources/networkpolicy.go index 28547e66..14354f7c 100644 --- a/controllers/resources/networkpolicy.go +++ b/internal/controller/resources/networkpolicy.go @@ -18,8 +18,9 @@ package resources import ( "context" "fmt" + "github.com/go-logr/logr" - "k8s.io/api/networking/v1" + v1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/controllers/resources/peerauthentication.go b/internal/controller/resources/peerauthentication.go similarity index 99% rename from controllers/resources/peerauthentication.go rename to internal/controller/resources/peerauthentication.go index b45bbb99..86a44c11 100644 --- a/controllers/resources/peerauthentication.go +++ b/internal/controller/resources/peerauthentication.go @@ -18,6 +18,7 @@ package resources import ( "context" "fmt" + "github.com/go-logr/logr" istiosecv1beta1 "istio.io/client-go/pkg/apis/security/v1beta1" "k8s.io/apimachinery/pkg/api/errors" diff --git a/controllers/resources/podmonitor.go b/internal/controller/resources/podmonitor.go similarity index 99% rename from controllers/resources/podmonitor.go rename to internal/controller/resources/podmonitor.go index 091a9751..f466814b 100644 --- a/controllers/resources/podmonitor.go +++ b/internal/controller/resources/podmonitor.go @@ -18,6 +18,7 @@ package resources import ( "context" "fmt" + "github.com/go-logr/logr" v1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "k8s.io/apimachinery/pkg/api/errors" diff --git a/controllers/resources/rolebinding.go b/internal/controller/resources/rolebinding.go similarity index 99% rename from controllers/resources/rolebinding.go rename to internal/controller/resources/rolebinding.go index ef863f00..e3d28ea8 100644 --- a/controllers/resources/rolebinding.go +++ b/internal/controller/resources/rolebinding.go @@ -18,6 +18,7 @@ package resources import ( "context" "fmt" + "github.com/go-logr/logr" v1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" diff --git a/controllers/resources/route.go b/internal/controller/resources/route.go similarity index 99% rename from controllers/resources/route.go rename to internal/controller/resources/route.go index 96a46fdf..f3d6d75b 100644 --- a/controllers/resources/route.go +++ b/internal/controller/resources/route.go @@ -18,6 +18,7 @@ package resources import ( "context" "fmt" + "github.com/go-logr/logr" v1 "github.com/openshift/api/route/v1" "k8s.io/apimachinery/pkg/api/errors" diff --git a/controllers/resources/secret.go b/internal/controller/resources/secret.go similarity index 100% rename from controllers/resources/secret.go rename to internal/controller/resources/secret.go diff --git a/controllers/resources/service.go b/internal/controller/resources/service.go similarity index 100% rename from controllers/resources/service.go rename to internal/controller/resources/service.go diff --git a/controllers/resources/serviceaccount.go b/internal/controller/resources/serviceaccount.go similarity index 99% rename from controllers/resources/serviceaccount.go rename to internal/controller/resources/serviceaccount.go index 659e7b43..40496742 100644 --- a/controllers/resources/serviceaccount.go +++ b/internal/controller/resources/serviceaccount.go @@ -18,6 +18,7 @@ package resources import ( "context" "fmt" + "github.com/go-logr/logr" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" diff --git a/controllers/resources/servicemeshmember.go b/internal/controller/resources/servicemeshmember.go similarity index 100% rename from controllers/resources/servicemeshmember.go rename to internal/controller/resources/servicemeshmember.go diff --git a/controllers/resources/servicemonitor.go b/internal/controller/resources/servicemonitor.go similarity index 99% rename from controllers/resources/servicemonitor.go rename to internal/controller/resources/servicemonitor.go index 6ba078dd..defbcff4 100644 --- a/controllers/resources/servicemonitor.go +++ b/internal/controller/resources/servicemonitor.go @@ -18,6 +18,7 @@ package resources import ( "context" "fmt" + "github.com/go-logr/logr" v1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "k8s.io/apimachinery/pkg/api/errors" diff --git a/controllers/resources/smmr.go b/internal/controller/resources/smmr.go similarity index 100% rename from controllers/resources/smmr.go rename to internal/controller/resources/smmr.go diff --git a/controllers/resources/telemetry.go b/internal/controller/resources/telemetry.go similarity index 99% rename from controllers/resources/telemetry.go rename to internal/controller/resources/telemetry.go index 5dbbeceb..23ae9654 100644 --- a/controllers/resources/telemetry.go +++ b/internal/controller/resources/telemetry.go @@ -18,6 +18,7 @@ package resources import ( "context" "fmt" + "github.com/go-logr/logr" telemetryv1alpha1 "istio.io/client-go/pkg/apis/telemetry/v1alpha1" "k8s.io/apimachinery/pkg/api/errors" diff --git a/controllers/resources/template/authconfig_anonymous.yaml b/internal/controller/resources/template/authconfig_anonymous.yaml similarity index 100% rename from controllers/resources/template/authconfig_anonymous.yaml rename to internal/controller/resources/template/authconfig_anonymous.yaml diff --git a/controllers/resources/template/authconfig_userdefined.yaml b/internal/controller/resources/template/authconfig_userdefined.yaml similarity index 100% rename from controllers/resources/template/authconfig_userdefined.yaml rename to internal/controller/resources/template/authconfig_userdefined.yaml diff --git a/controllers/resources/unit_test.go b/internal/controller/resources/unit_test.go similarity index 96% rename from controllers/resources/unit_test.go rename to internal/controller/resources/unit_test.go index 2ea5c577..54dc7759 100644 --- a/controllers/resources/unit_test.go +++ b/internal/controller/resources/unit_test.go @@ -20,7 +20,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - //+kubebuilder:scaffold:imports + // +kubebuilder:scaffold:imports ) // These tests use Ginkgo (BDD-style Go testing framework). Refer to diff --git a/internal/controller/serving/inferenceservice_controller.go b/internal/controller/serving/inferenceservice_controller.go new file mode 100644 index 00000000..afb8de8c --- /dev/null +++ b/internal/controller/serving/inferenceservice_controller.go @@ -0,0 +1,288 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package serving + +import ( + "context" + "errors" + + "github.com/go-logr/logr" + kservev1alpha1 "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" + kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" + routev1 "github.com/openshift/api/route/v1" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + authv1 "k8s.io/api/rbac/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" + "github.com/opendatahub-io/odh-model-controller/internal/controller/serving/reconcilers" + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" +) + +// InferenceServiceReconciler reconciles a InferenceService object +type InferenceServiceReconciler struct { + client.Client + Scheme *runtime.Scheme + + clientReader client.Reader + bearerToken string + + MeshDisabled bool + ModelRegistryEnabled bool + modelRegistrySkipTls bool + mmISVCReconciler *reconcilers.ModelMeshInferenceServiceReconciler + kserveServerlessISVCReconciler *reconcilers.KserveServerlessInferenceServiceReconciler + kserveRawISVCReconciler *reconcilers.KserveRawInferenceServiceReconciler +} + +func NewInferenceServiceReconciler(setupLog logr.Logger, client client.Client, scheme *runtime.Scheme, clientReader client.Reader, + kClient kubernetes.Interface, meshDisabled bool, modelRegistryReconcileEnabled, modelRegistrySkipTls bool, bearerToken string) *InferenceServiceReconciler { + isvcReconciler := &InferenceServiceReconciler{ + Client: client, + Scheme: scheme, + clientReader: clientReader, + MeshDisabled: meshDisabled, + ModelRegistryEnabled: modelRegistryReconcileEnabled, + modelRegistrySkipTls: modelRegistrySkipTls, + mmISVCReconciler: reconcilers.NewModelMeshInferenceServiceReconciler(client), + kserveServerlessISVCReconciler: reconcilers.NewKServeServerlessInferenceServiceReconciler(client, clientReader, kClient), + kserveRawISVCReconciler: reconcilers.NewKServeRawInferenceServiceReconciler(client), + bearerToken: bearerToken, + } + + if modelRegistryReconcileEnabled { + setupLog.Info("Model registry inference service reconciliation enabled.") + } else { + setupLog.Info("Model registry inference service reconciliation disabled. To enable model registry " + + "reconciliation for InferenceService, please provide --model-registry-inference-reconcile flag.") + } + + return isvcReconciler +} + +// +kubebuilder:rbac:groups=serving.kserve.io,resources=inferenceservices,verbs=get;list;watch;update +// +kubebuilder:rbac:groups=serving.kserve.io,resources=inferenceservices/finalizers,verbs=get;list;watch;update + +// +kubebuilder:rbac:groups=networking.istio.io,resources=virtualservices,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=networking.istio.io,resources=virtualservices/finalizers,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=networking.istio.io,resources=gateways,verbs=get;list;watch;update;patch +// +kubebuilder:rbac:groups=security.istio.io,resources=peerauthentications,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=security.istio.io,resources=authorizationpolicies,verbs=get;list +// +kubebuilder:rbac:groups=telemetry.istio.io,resources=telemetries,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=maistra.io,resources=servicemeshmembers,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=maistra.io,resources=servicemeshmemberrolls,verbs=get;list;watch +// +kubebuilder:rbac:groups=maistra.io,resources=servicemeshcontrolplanes,verbs=get;list;watch;use +// +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=route.openshift.io,resources=routes/custom-host,verbs=create +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterrolebindings;rolebindings,verbs=get;list;watch;create;update;patch;watch;delete +// +kubebuilder:rbac:groups=networking.k8s.io,resources=networkpolicies,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch +// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors;podmonitors,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=extensions,resources=ingresses,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=namespaces;pods;endpoints,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups="",resources=secrets;configmaps;serviceaccounts;services,verbs=get;list;watch;create;update;delete;patch +// +kubebuilder:rbac:groups=authorino.kuadrant.io,resources=authconfigs,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=datasciencecluster.opendatahub.io,resources=datascienceclusters,verbs=get;list;watch +// +kubebuilder:rbac:groups=dscinitialization.opendatahub.io,resources=dscinitializations,verbs=get;list;watch + +// Reconcile performs the reconciling of the Openshift objects for a Kubeflow +// InferenceService. +func (r *InferenceServiceReconciler) ReconcileServing(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + // Initialize logger format + logger := log.FromContext(ctx).WithValues("InferenceService", req.Name, "namespace", req.Namespace) + // Get the InferenceService object when a reconciliation event is triggered (create, + // update, delete) + isvc := &kservev1beta1.InferenceService{} + err := r.Client.Get(ctx, req.NamespacedName, isvc) + if err != nil && apierrs.IsNotFound(err) { + logger.Info("Stop InferenceService reconciliation") + // InferenceService not found, so we check for any other inference services that might be using Kserve/ModelMesh + // If none are found, we delete the common namespace-scoped resources that were created for Kserve/ModelMesh. + err1 := r.DeleteResourcesIfNoIsvcExists(ctx, logger, req.Namespace) + if err1 != nil { + logger.Error(err1, "Unable to clean up resources") + return ctrl.Result{}, err1 + } + return ctrl.Result{}, nil + } else if err != nil { + logger.Error(err, "Unable to fetch the InferenceService") + return ctrl.Result{}, err + } + + if isvc.GetDeletionTimestamp() != nil { + return reconcile.Result{}, r.onDeletion(ctx, logger, isvc) + } + + // Check what deployment mode is used by the InferenceService. We have differing reconciliation logic for Kserve and ModelMesh + IsvcDeploymentMode, err := utils.GetDeploymentModeForIsvc(ctx, r.Client, isvc) + if err != nil { + return ctrl.Result{}, err + } + switch IsvcDeploymentMode { + case constants.ModelMesh: + logger.Info("Reconciling InferenceService for ModelMesh") + err = r.mmISVCReconciler.Reconcile(ctx, logger, isvc) + case constants.Serverless: + logger.Info("Reconciling InferenceService for Kserve in mode Serverless") + err = r.kserveServerlessISVCReconciler.Reconcile(ctx, logger, isvc) + case constants.RawDeployment: + logger.Info("Reconciling InferenceService for Kserve in mode RawDeployment") + err = r.kserveRawISVCReconciler.Reconcile(ctx, logger, isvc) + } + + return ctrl.Result{}, err +} + +// Reconcile is the top-level function to run the different integrations with ODH platform: +// - Model Registry integration (opt-in via CLI flag) +// - OpenShift and other ODH features, like Data Connections, Routes and certificate trust. +func (r *InferenceServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + reconcileResult, reconcileErr := r.ReconcileServing(ctx, req) + + if r.ModelRegistryEnabled { + mrReconciler := reconcilers.NewModelRegistryInferenceServiceReconciler( + r.Client, + log.FromContext(ctx).WithName("controllers").WithName("ModelRegistryInferenceService"), + r.modelRegistrySkipTls, + r.bearerToken, + ) + + mrResult, mrErr := mrReconciler.Reconcile(ctx, req) + + if mrResult.Requeue { + reconcileResult.Requeue = true + } + + if mrResult.RequeueAfter > 0 && (reconcileResult.RequeueAfter == 0 || mrResult.RequeueAfter < reconcileResult.RequeueAfter) { + reconcileResult.RequeueAfter = mrResult.RequeueAfter + } + + reconcileErr = errors.Join(reconcileErr, mrErr) + } + + return reconcileResult, reconcileErr +} + +// SetupWithManager sets up the controller with the Manager. +func (r *InferenceServiceReconciler) SetupWithManager(mgr ctrl.Manager) error { + builder := ctrl.NewControllerManagedBy(mgr). + For(&kservev1beta1.InferenceService{}). + Owns(&kservev1alpha1.ServingRuntime{}). + Owns(&corev1.Namespace{}). + Owns(&routev1.Route{}). + Owns(&corev1.ServiceAccount{}). + Owns(&corev1.Service{}). + Owns(&corev1.ConfigMap{}). + Owns(&corev1.Secret{}). + Owns(&authv1.ClusterRoleBinding{}). + Owns(&networkingv1.NetworkPolicy{}). + Owns(&monitoringv1.ServiceMonitor{}). + Owns(&monitoringv1.PodMonitor{}). + Named("inferenceservice"). + Watches(&kservev1alpha1.ServingRuntime{}, + handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []reconcile.Request { + logger := log.FromContext(ctx) + logger.Info("Reconcile event triggered by serving runtime: " + o.GetName()) + inferenceServicesList := &kservev1beta1.InferenceServiceList{} + opts := []client.ListOption{client.InNamespace(o.GetNamespace())} + + // Todo: Get only Inference Services that are deploying on the specific serving runtime + err := r.Client.List(ctx, inferenceServicesList, opts...) + if err != nil { + logger.Info("Error getting list of inference services for namespace") + return []reconcile.Request{} + } + + if len(inferenceServicesList.Items) == 0 { + logger.Info("No InferenceServices found for Serving Runtime: " + o.GetName()) + return []reconcile.Request{} + } + + reconcileRequests := make([]reconcile.Request, 0, len(inferenceServicesList.Items)) + for _, inferenceService := range inferenceServicesList.Items { + reconcileRequests = append(reconcileRequests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: inferenceService.Name, + Namespace: inferenceService.Namespace, + }, + }) + } + return reconcileRequests + })). + Watches(&corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []reconcile.Request { + logger := log.FromContext(ctx) + logger.Info("Reconcile event triggered by Secret: " + o.GetName()) + isvc := &kservev1beta1.InferenceService{} + err := r.Client.Get(ctx, types.NamespacedName{Name: o.GetName(), Namespace: o.GetNamespace()}, isvc) + if err != nil { + if apierrs.IsNotFound(err) { + return []reconcile.Request{} + } + logger.Error(err, "Error getting the inferenceService", "name", o.GetName()) + return []reconcile.Request{} + } + + return []reconcile.Request{ + {NamespacedName: types.NamespacedName{Name: o.GetName(), Namespace: o.GetNamespace()}}, + } + })) + + return builder.Complete(r) +} + +// general clean-up, mostly resources in different namespaces from kservev1beta1.InferenceService +func (r *InferenceServiceReconciler) onDeletion(ctx context.Context, log logr.Logger, inferenceService *kservev1beta1.InferenceService) error { + log.V(1).Info("Running cleanup logic") + + IsvcDeploymentMode, err := utils.GetDeploymentModeForIsvc(ctx, r.Client, inferenceService) + if err != nil { + log.V(1).Error(err, "Could not determine deployment mode for ISVC. Some resources related to the inferenceservice might not be deleted.") + } + if IsvcDeploymentMode == constants.Serverless { + log.V(1).Info("Deleting kserve inference resource (Serverless Mode)") + return r.kserveServerlessISVCReconciler.OnDeletionOfKserveInferenceService(ctx, log, inferenceService) + } + if IsvcDeploymentMode == constants.RawDeployment { + log.V(1).Info("Deleting kserve inference resource (RawDeployment Mode)") + return r.kserveRawISVCReconciler.OnDeletionOfKserveInferenceService(ctx, log, inferenceService) + } + return nil +} + +func (r *InferenceServiceReconciler) DeleteResourcesIfNoIsvcExists(ctx context.Context, log logr.Logger, namespace string) error { + if err := r.kserveServerlessISVCReconciler.CleanupNamespaceIfNoKserveIsvcExists(ctx, log, namespace); err != nil { + return err + } + if err := r.mmISVCReconciler.DeleteModelMeshResourcesIfNoMMIsvcExists(ctx, log, namespace); err != nil { + return err + } + if err := r.kserveRawISVCReconciler.CleanupNamespaceIfNoKserveIsvcExists(ctx, log, namespace); err != nil { + return err + } + return nil +} diff --git a/internal/controller/serving/inferenceservice_controller_test.go b/internal/controller/serving/inferenceservice_controller_test.go new file mode 100644 index 00000000..95f20cc9 --- /dev/null +++ b/internal/controller/serving/inferenceservice_controller_test.go @@ -0,0 +1,1359 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package serving + +import ( + "context" + "errors" + "fmt" + "reflect" + "strings" + "time" + + kservev1alpha1 "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" + kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" + authorinov1beta2 "github.com/kuadrant/authorino/api/v1beta2" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + gomegatypes "github.com/onsi/gomega/types" + routev1 "github.com/openshift/api/route/v1" + istioclientv1beta1 "istio.io/client-go/pkg/apis/networking/v1beta1" + istiosecv1b1 "istio.io/client-go/pkg/apis/security/v1beta1" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" + k8sErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/dynamic" + "knative.dev/pkg/apis" + duckv1 "knative.dev/pkg/apis/duck/v1" + maistrav1 "maistra.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/comparators" + "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" + testutils "github.com/opendatahub-io/odh-model-controller/test/utils" +) + +const ( + KserveOvmsInferenceServiceName = "example-onnx-mnist" + UnsupportedMetricsInferenceServiceName = "sklearn-v2-iris" + NilRuntimeInferenceServiceName = "sklearn-v2-iris-no-runtime" + NilModelInferenceServiceName = "custom-runtime" + + UnsupportedMetricsInferenceServicePath = "./testdata/deploy/kserve-unsupported-metrics-inference-service.yaml" + UnsupprtedMetricsServingRuntimePath = "./testdata/deploy/kserve-unsupported-metrics-serving-runtime.yaml" + NilRuntimeInferenceServicePath = "./testdata/deploy/kserve-nil-runtime-inference-service.yaml" + NilModelInferenceServicePath = "./testdata/deploy/kserve-nil-model-inference-service.yaml" + testIsvcSvcPath = "./testdata/servingcert-service/test-isvc-svc.yaml" + kserveLocalGatewayPath = "./testdata/gateway/kserve-local-gateway.yaml" + testIsvcSvcSecretPath = "./testdata/gateway/test-isvc-svc-secret.yaml" +) + +var _ = Describe("InferenceService Controller", func() { + Describe("Openshift KServe integrations", func() { + + When("creating a Kserve ServiceRuntime & InferenceService", func() { + var testNs string + + BeforeEach(func() { + ctx := context.Background() + testNamespace := testutils.Namespaces.Create(ctx, k8sClient) + testNs = testNamespace.Name + + inferenceServiceConfig := &corev1.ConfigMap{} + Expect(testutils.ConvertToStructuredResource(InferenceServiceConfigPath1, inferenceServiceConfig)).To(Succeed()) + if err := k8sClient.Create(ctx, inferenceServiceConfig); err != nil && !k8sErrors.IsAlreadyExists(err) { + Fail(err.Error()) + } + + servingRuntime := &kservev1alpha1.ServingRuntime{} + Expect(testutils.ConvertToStructuredResource(KserveServingRuntimePath1, servingRuntime)).To(Succeed()) + servingRuntime.SetNamespace(testNs) + if err := k8sClient.Create(ctx, servingRuntime); err != nil && !k8sErrors.IsAlreadyExists(err) { + Fail(err.Error()) + } + }) + + It("With Kserve InferenceService a Route be created", func() { + _, meshNamespace := utils.GetIstioControlPlaneName(ctx, k8sClient) + inferenceService := &kservev1beta1.InferenceService{} + err := testutils.ConvertToStructuredResource(KserveInferenceServicePath1, inferenceService) + Expect(err).NotTo(HaveOccurred()) + inferenceService.SetNamespace(testNs) + + Expect(k8sClient.Create(ctx, inferenceService)).Should(Succeed()) + + By("By checking that the controller has not created the Route") + Consistently(func() error { + route := &routev1.Route{} + key := types.NamespacedName{Name: getKServeRouteName(inferenceService), Namespace: meshNamespace} + err = k8sClient.Get(ctx, key, route) + return err + }, timeout, interval).Should(HaveOccurred()) + + deployedInferenceService := &kservev1beta1.InferenceService{} + err = k8sClient.Get(ctx, types.NamespacedName{Name: inferenceService.Name, Namespace: inferenceService.Namespace}, deployedInferenceService) + Expect(err).NotTo(HaveOccurred()) + + url, err := apis.ParseURL("https://example-onnx-mnist-default.test.com") + Expect(err).NotTo(HaveOccurred()) + deployedInferenceService.Status.URL = url + + err = k8sClient.Status().Update(ctx, deployedInferenceService) + Expect(err).NotTo(HaveOccurred()) + + By("By checking that the controller has created the Route") + Eventually(func() error { + route := &routev1.Route{} + key := types.NamespacedName{Name: getKServeRouteName(inferenceService), Namespace: meshNamespace} + err = k8sClient.Get(ctx, key, route) + return err + }, timeout, interval).Should(Succeed()) + }) + It("With a new Kserve InferenceService, serving cert annotation should be added to the runtime Service object.", func() { + // We need to stub the cluster state and indicate where is istio namespace (reusing authConfig test data) + if dsciErr := createDSCI(DSCIWithoutAuthorization); dsciErr != nil && !k8sErrors.IsAlreadyExists(dsciErr) { + Fail(dsciErr.Error()) + } + // Create a new InferenceService + inferenceService := &kservev1beta1.InferenceService{} + err := testutils.ConvertToStructuredResource(KserveInferenceServicePath1, inferenceService) + Expect(err).NotTo(HaveOccurred()) + inferenceService.SetNamespace(testNs) + Expect(k8sClient.Create(ctx, inferenceService)).Should(Succeed()) + // Update the URL of the InferenceService to indicate it is ready. + deployedInferenceService := &kservev1beta1.InferenceService{} + err = k8sClient.Get(ctx, types.NamespacedName{Name: inferenceService.Name, Namespace: inferenceService.Namespace}, deployedInferenceService) + Expect(err).NotTo(HaveOccurred()) + // url, err := apis.ParseURL("https://example-onnx-mnist-default.test.com") + Expect(err).NotTo(HaveOccurred()) + newAddress := &duckv1.Addressable{ + URL: apis.HTTPS("example-onnx-mnist-default.test.com"), + } + deployedInferenceService.Status.Address = newAddress + err = k8sClient.Status().Update(ctx, deployedInferenceService) + Expect(err).NotTo(HaveOccurred()) + // Stub: Create a Kserve Service, which must be created by the KServe operator. + svc := &corev1.Service{} + err = testutils.ConvertToStructuredResource(testIsvcSvcPath, svc) + Expect(err).NotTo(HaveOccurred()) + svc.SetNamespace(inferenceService.Namespace) + Expect(k8sClient.Create(ctx, svc)).Should(Succeed()) + err = k8sClient.Status().Update(ctx, deployedInferenceService) + Expect(err).NotTo(HaveOccurred()) + // isvcService, err := waitForService(cli, testNs, inferenceService.Name, 5, 2*time.Second) + // Expect(err).NotTo(HaveOccurred()) + + isvcService := &corev1.Service{} + Eventually(func() error { + err := k8sClient.Get(ctx, client.ObjectKey{Namespace: inferenceService.Namespace, Name: inferenceService.Name}, isvcService) + if err != nil { + return err + } + if isvcService.Annotations == nil || isvcService.Annotations[constants.ServingCertAnnotationKey] == "" { + + return fmt.Errorf("Annotation[constants.ServingCertAnnotationKey] is not added yet") + } + return nil + }, timeout, interval).Should(Succeed()) + + Expect(isvcService.Annotations[constants.ServingCertAnnotationKey]).Should(Equal(inferenceService.Name)) + }) + + It("should create a secret for runtime and update kserve local gateway in the istio-system namespace", func() { + // We need to stub the cluster state and indicate where is istio namespace (reusing authConfig test data) + if dsciErr := createDSCI(DSCIWithoutAuthorization); dsciErr != nil && !k8sErrors.IsAlreadyExists(dsciErr) { + Fail(dsciErr.Error()) + } + // Stub: Create a kserve-local-gateway, which must be created by the OpenDataHub operator. + kserveLocalGateway := &istioclientv1beta1.Gateway{} + err := testutils.ConvertToStructuredResource(kserveLocalGatewayPath, kserveLocalGateway) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient.Create(ctx, kserveLocalGateway)).Should(Succeed()) + + // Stub: Create a certificate Secret, which must be created by the openshift service-ca operator. + secret := &corev1.Secret{} + err = testutils.ConvertToStructuredResource(testIsvcSvcSecretPath, secret) + Expect(err).NotTo(HaveOccurred()) + secret.SetNamespace(testNs) + Expect(k8sClient.Create(ctx, secret)).Should(Succeed()) + + // Create a new InferenceService + inferenceService := &kservev1beta1.InferenceService{} + err = testutils.ConvertToStructuredResource(KserveInferenceServicePath1, inferenceService) + Expect(err).NotTo(HaveOccurred()) + inferenceService.SetNamespace(testNs) + + Expect(k8sClient.Create(ctx, inferenceService)).Should(Succeed()) + + // Update the URL of the InferenceService to indicate it is ready. + deployedInferenceService := &kservev1beta1.InferenceService{} + err = k8sClient.Get(ctx, types.NamespacedName{Name: inferenceService.Name, Namespace: inferenceService.Namespace}, deployedInferenceService) + Expect(err).NotTo(HaveOccurred()) + + newAddress := &duckv1.Addressable{ + URL: apis.HTTPS("example-onnx-mnist-default.test.com"), + } + deployedInferenceService.Status.Address = newAddress + + err = k8sClient.Status().Update(ctx, deployedInferenceService) + Expect(err).NotTo(HaveOccurred()) + + _, meshNamespace := utils.GetIstioControlPlaneName(ctx, k8sClient) + + // Verify that the certificate secret is created in the istio-system namespace. + Eventually(func() error { + secret := &corev1.Secret{} + return k8sClient.Get(ctx, client.ObjectKey{Namespace: meshNamespace, Name: fmt.Sprintf("%s-%s", inferenceService.Name, inferenceService.Namespace)}, secret) + }, timeout, interval).Should(Succeed()) + + // Verify that the gateway is updated in the istio-system namespace. + var gateway *istioclientv1beta1.Gateway + Eventually(func() error { + gateway, err = waitForUpdatedGatewayCompletion(k8sClient, "add", meshNamespace, constants.KServeGatewayName, inferenceService.Name) + return err + }, timeout, interval).Should(Succeed()) + + // Ensure that the server is successfully added to the KServe local gateway within the istio-system namespace. + targetServerExist := hasServerFromGateway(gateway, fmt.Sprintf("%s-%s", "https", inferenceService.Name)) + Expect(targetServerExist).Should(BeTrue()) + }) + + It("should create required network policies when KServe is used", func() { + // given + inferenceService := &kservev1beta1.InferenceService{} + Expect(testutils.ConvertToStructuredResource(KserveInferenceServicePath1, inferenceService)).To(Succeed()) + inferenceService.SetNamespace(testNs) + + // when + Expect(k8sClient.Create(ctx, inferenceService)).Should(Succeed()) + + // then + By("ensuring that the controller has created required network policies") + networkPolicies := &v1.NetworkPolicyList{} + Eventually(func() []v1.NetworkPolicy { + err := k8sClient.List(ctx, networkPolicies, client.InNamespace(inferenceService.Namespace)) + if err != nil { + Fail(err.Error()) + } + return networkPolicies.Items + }, timeout, interval).Should( + ContainElements( + withMatchingNestedField("ObjectMeta.Name", Equal("allow-from-openshift-monitoring-ns")), + withMatchingNestedField("ObjectMeta.Name", Equal("allow-openshift-ingress")), + withMatchingNestedField("ObjectMeta.Name", Equal("allow-from-opendatahub-ns")), + ), + ) + }) + }) + + Context("when there is a existing inferenceService", func() { + var testNs string + var isvcName string + + BeforeEach(func() { + ctx := context.Background() + testNamespace := testutils.Namespaces.Create(ctx, k8sClient) + testNs = testNamespace.Name + _, meshNamespace := utils.GetIstioControlPlaneName(ctx, k8sClient) + + inferenceServiceConfig := &corev1.ConfigMap{} + Expect(testutils.ConvertToStructuredResource(InferenceServiceConfigPath1, inferenceServiceConfig)).To(Succeed()) + if err := k8sClient.Create(ctx, inferenceServiceConfig); err != nil && !k8sErrors.IsAlreadyExists(err) { + Fail(err.Error()) + } + + // We need to stub the cluster state and indicate where is istio namespace (reusing authConfig test data) + if dsciErr := createDSCI(DSCIWithoutAuthorization); dsciErr != nil && !k8sErrors.IsAlreadyExists(dsciErr) { + Fail(dsciErr.Error()) + } + + servingRuntime := &kservev1alpha1.ServingRuntime{} + Expect(testutils.ConvertToStructuredResource(KserveServingRuntimePath1, servingRuntime)).To(Succeed()) + if err := k8sClient.Create(ctx, servingRuntime); err != nil && !k8sErrors.IsAlreadyExists(err) { + Fail(err.Error()) + } + + // Stub: Create a kserve-local-gateway, which must be created by the OpenDataHub operator. + kserveLocalGateway := &istioclientv1beta1.Gateway{} + err := testutils.ConvertToStructuredResource(kserveLocalGatewayPath, kserveLocalGateway) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient.Create(ctx, kserveLocalGateway)).Should(Succeed()) + + // Stub: Create a certificate Secret, which must be created by the openshift service-ca operator. + secret := &corev1.Secret{} + err = testutils.ConvertToStructuredResource(testIsvcSvcSecretPath, secret) + Expect(err).NotTo(HaveOccurred()) + secret.SetNamespace(testNs) + Expect(k8sClient.Create(ctx, secret)).Should(Succeed()) + + // Create a new InferenceService + inferenceService := &kservev1beta1.InferenceService{} + err = testutils.ConvertToStructuredResource(KserveInferenceServicePath1, inferenceService) + Expect(err).NotTo(HaveOccurred()) + inferenceService.SetNamespace(testNs) + // Ensure the Delete method is called when the InferenceService (ISVC) is deleted. + inferenceService.SetFinalizers([]string{"finalizer.inferenceservice"}) + + Expect(k8sClient.Create(ctx, inferenceService)).Should(Succeed()) + isvcName = inferenceService.Name + + // Update the URL of the InferenceService to indicate it is ready. + deployedInferenceService := &kservev1beta1.InferenceService{} + err = k8sClient.Get(ctx, types.NamespacedName{Name: inferenceService.Name, Namespace: testNs}, deployedInferenceService) + Expect(err).NotTo(HaveOccurred()) + + newAddress := &duckv1.Addressable{ + URL: apis.HTTPS("example-onnx-mnist-default.test.com"), + } + deployedInferenceService.Status.Address = newAddress + + err = k8sClient.Status().Update(ctx, deployedInferenceService) + Expect(err).NotTo(HaveOccurred()) + + // Verify that the certificate secret is created in the istio-system namespace. + Eventually(func() error { + secret := &corev1.Secret{} + return k8sClient.Get(ctx, types.NamespacedName{Name: inferenceService.Name, Namespace: inferenceService.Namespace}, secret) + }, timeout, interval).Should(Succeed()) + + Eventually(func() error { + return k8sClient.Get(ctx, client.ObjectKey{Namespace: meshNamespace, Name: fmt.Sprintf("%s-%s", inferenceService.Name, inferenceService.Namespace)}, secret) + }, timeout, interval).Should(Succeed()) + + // Verify that the gateway is updated in the istio-system namespace. + var gateway *istioclientv1beta1.Gateway + Eventually(func() error { + gateway, err = waitForUpdatedGatewayCompletion(k8sClient, "add", meshNamespace, constants.KServeGatewayName, inferenceService.Name) + return err + }, timeout, interval).Should(Succeed()) + + // Ensure that the server is successfully added to the KServe local gateway within the istio-system namespace. + targetServerExist := hasServerFromGateway(gateway, fmt.Sprintf("%s-%s", "https", inferenceService.Name)) + Expect(targetServerExist).Should(BeTrue()) + }) + + When("serving cert Secret is rotated", func() { + It("should re-sync serving cert Secret to istio-system", func() { + deployedInferenceService := &kservev1beta1.InferenceService{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: isvcName, Namespace: testNs}, deployedInferenceService) + Expect(err).NotTo(HaveOccurred()) + + // Get source secret + srcSecret := &corev1.Secret{} + err = k8sClient.Get(ctx, client.ObjectKey{Namespace: testNs, Name: deployedInferenceService.Name}, srcSecret) + Expect(err).NotTo(HaveOccurred()) + + // Update source secret + updatedDataString := "updateData" + srcSecret.Data["tls.crt"] = []byte(updatedDataString) + srcSecret.Data["tls.key"] = []byte(updatedDataString) + Expect(k8sClient.Update(ctx, srcSecret)).Should(Succeed()) + + // Get destination secret + err = k8sClient.Get(ctx, client.ObjectKey{Namespace: testNs, Name: deployedInferenceService.Name}, srcSecret) + Expect(err).NotTo(HaveOccurred()) + + // Verify that the certificate secret in the istio-system namespace is updated. + _, meshNamespace := utils.GetIstioControlPlaneName(ctx, k8sClient) + destSecret := &corev1.Secret{} + Eventually(func() error { + Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: meshNamespace, Name: fmt.Sprintf("%s-%s", deployedInferenceService.Name, deployedInferenceService.Namespace)}, destSecret)).Should(Succeed()) + if string(destSecret.Data["tls.crt"]) != updatedDataString { + return fmt.Errorf("destSecret is not updated yet") + } + return nil + }, timeout, interval).Should(Succeed()) + + Expect(destSecret.Data).To(Equal(srcSecret.Data)) + }) + }) + + When("infereceService is deleted", func() { + It("should remove the Server from the kserve local gateway in istio-system and delete the created Secret", func() { + // Delete the existing ISVC + deployedInferenceService := &kservev1beta1.InferenceService{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: isvcName, Namespace: testNs}, deployedInferenceService) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient.Delete(ctx, deployedInferenceService)).Should(Succeed()) + + _, meshNamespace := utils.GetIstioControlPlaneName(ctx, k8sClient) + + // Verify that the gateway is updated in the istio-system namespace. + var gateway *istioclientv1beta1.Gateway + Eventually(func() error { + gateway, err = waitForUpdatedGatewayCompletion(k8sClient, "delete", meshNamespace, constants.KServeGatewayName, isvcName) + return err + }, timeout, interval).Should(Succeed()) + + // Ensure that the server is successfully removed from the KServe local gateway within the istio-system namespace. + targetServerExist := hasServerFromGateway(gateway, isvcName) + Expect(targetServerExist).Should(BeFalse()) + + // Ensure that the synced Secret is successfully deleted within the istio-system namespace. + secret := &corev1.Secret{} + Eventually(func() error { + return k8sClient.Get(ctx, client.ObjectKey{Namespace: meshNamespace, Name: fmt.Sprintf("%s-%s", isvcName, meshNamespace)}, secret) + }, timeout, interval).ShouldNot(Succeed()) + }) + }) + }) + + }) + + Describe("Openshift ModelMesh integrations", func() { + + When("creating a ServiceRuntime & InferenceService with 'enable-route' enabled", func() { + + BeforeEach(func() { + ctx := context.Background() + + servingRuntime1 := &kservev1alpha1.ServingRuntime{} + err := testutils.ConvertToStructuredResource(ServingRuntimePath1, servingRuntime1) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient.Create(ctx, servingRuntime1)).Should(Succeed()) + + servingRuntime2 := &kservev1alpha1.ServingRuntime{} + err = testutils.ConvertToStructuredResource(ServingRuntimePath2, servingRuntime2) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient.Create(ctx, servingRuntime2)).Should(Succeed()) + }) + + It("when InferenceService specifies a runtime, should create a Route to expose the traffic externally", func() { + inferenceService := &kservev1beta1.InferenceService{} + err := testutils.ConvertToStructuredResource(InferenceService1, inferenceService) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient.Create(ctx, inferenceService)).Should(Succeed()) + + By("By checking that the controller has created the Route") + + route := &routev1.Route{} + Eventually(func() error { + key := types.NamespacedName{Name: inferenceService.Name, Namespace: inferenceService.Namespace} + return k8sClient.Get(ctx, key, route) + }, timeout, interval).ShouldNot(HaveOccurred()) + + expectedRoute := &routev1.Route{} + err = testutils.ConvertToStructuredResource(ExpectedRoutePath, expectedRoute) + Expect(err).NotTo(HaveOccurred()) + + Expect(comparators.GetMMRouteComparator()(route, expectedRoute)).Should(BeTrue()) + }) + + It("when InferenceService does not specifies a runtime, should automatically pick a runtime and create a Route", func() { + inferenceService := &kservev1beta1.InferenceService{} + err := testutils.ConvertToStructuredResource(InferenceServiceNoRuntime, inferenceService) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient.Create(ctx, inferenceService)).Should(Succeed()) + + route := &routev1.Route{} + Eventually(func() error { + key := types.NamespacedName{Name: inferenceService.Name, Namespace: inferenceService.Namespace} + return k8sClient.Get(ctx, key, route) + }, timeout, interval).ShouldNot(HaveOccurred()) + + expectedRoute := &routev1.Route{} + err = testutils.ConvertToStructuredResource(ExpectedRouteNoRuntimePath, expectedRoute) + Expect(err).NotTo(HaveOccurred()) + + Expect(comparators.GetMMRouteComparator()(route, expectedRoute)).Should(BeTrue()) + }) + }) + }) + + Describe("Mesh reconciler", func() { + var testNs string + + createInferenceService := func(namespace, name string) *kservev1beta1.InferenceService { + inferenceService := &kservev1beta1.InferenceService{} + err := testutils.ConvertToStructuredResource(KserveInferenceServicePath1, inferenceService) + Expect(err).NotTo(HaveOccurred()) + inferenceService.SetNamespace(namespace) + if len(name) != 0 { + inferenceService.Name = name + } + Expect(k8sClient.Create(ctx, inferenceService)).Should(Succeed()) + + return inferenceService + } + + expectOwnedSmmCreated := func(namespace string) { + Eventually(func() error { + smm := &maistrav1.ServiceMeshMember{} + key := types.NamespacedName{Name: constants.ServiceMeshMemberName, Namespace: namespace} + err := k8sClient.Get(ctx, key, smm) + return err + }, timeout, interval).Should(Succeed()) + } + + createUserOwnedMeshEnrolment := func(namespace string) *maistrav1.ServiceMeshMember { + controlPlaneName, meshNamespace := utils.GetIstioControlPlaneName(ctx, k8sClient) + smm := &maistrav1.ServiceMeshMember{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.ServiceMeshMemberName, + Namespace: namespace, + Labels: nil, + Annotations: nil, + }, + Spec: maistrav1.ServiceMeshMemberSpec{ + ControlPlaneRef: maistrav1.ServiceMeshControlPlaneRef{ + Name: controlPlaneName, + Namespace: meshNamespace, + }}, + Status: maistrav1.ServiceMeshMemberStatus{}, + } + Expect(k8sClient.Create(ctx, smm)).Should(Succeed()) + + return smm + } + + BeforeEach(func() { + testNamespace := testutils.Namespaces.Create(ctx, k8sClient) + testNs = testNamespace.Name + + inferenceServiceConfig := &corev1.ConfigMap{} + Expect(testutils.ConvertToStructuredResource(InferenceServiceConfigPath1, inferenceServiceConfig)).To(Succeed()) + if err := k8sClient.Create(ctx, inferenceServiceConfig); err != nil && !k8sErrors.IsAlreadyExists(err) { + Fail(err.Error()) + } + }) + + When("deploying the first model in a namespace", func() { + It("if the namespace is not part of the service mesh, it should enroll the namespace to the mesh", func() { + inferenceService := createInferenceService(testNs, "") + expectOwnedSmmCreated(inferenceService.Namespace) + }) + + It("if the namespace is already enrolled to the service mesh by the user, it should not modify the enrollment", func() { + smm := createUserOwnedMeshEnrolment(testNs) + inferenceService := createInferenceService(testNs, "") + + Consistently(func() bool { + actualSmm := &maistrav1.ServiceMeshMember{} + key := types.NamespacedName{Name: constants.ServiceMeshMemberName, Namespace: inferenceService.Namespace} + err := k8sClient.Get(ctx, key, actualSmm) + return err == nil && reflect.DeepEqual(actualSmm, smm) + }).Should(BeTrue()) + }) + + It("if the namespace is already enrolled to some other control plane, it should anyway not modify the enrollment", func() { + smm := &maistrav1.ServiceMeshMember{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.ServiceMeshMemberName, + Namespace: testNs, + Labels: nil, + Annotations: nil, + }, + Spec: maistrav1.ServiceMeshMemberSpec{ + ControlPlaneRef: maistrav1.ServiceMeshControlPlaneRef{ + Name: "random-control-plane-vbfr238497", + Namespace: "random-namespace-a234h", + }}, + Status: maistrav1.ServiceMeshMemberStatus{}, + } + Expect(k8sClient.Create(ctx, smm)).Should(Succeed()) + + inferenceService := createInferenceService(testNs, "") + + Consistently(func() bool { + actualSmm := &maistrav1.ServiceMeshMember{} + key := types.NamespacedName{Name: constants.ServiceMeshMemberName, Namespace: inferenceService.Namespace} + err := k8sClient.Get(ctx, key, actualSmm) + return err == nil && reflect.DeepEqual(actualSmm, smm) + }).Should(BeTrue()) + }) + }) + + When("deleting the last model in a namespace", func() { + It("it should remove the owned service mesh enrolment", func() { + inferenceService := createInferenceService(testNs, "") + expectOwnedSmmCreated(inferenceService.Namespace) + + Expect(k8sClient.Delete(ctx, inferenceService)).Should(Succeed()) + Eventually(func() error { + smm := &maistrav1.ServiceMeshMember{} + key := types.NamespacedName{Name: constants.ServiceMeshMemberName, Namespace: inferenceService.Namespace} + err := k8sClient.Get(ctx, key, smm) + return err + }, timeout, interval).ShouldNot(Succeed()) + }) + + It("it should not remove a user-owned service mesh enrolment", func() { + createUserOwnedMeshEnrolment(testNs) + inferenceService := createInferenceService(testNs, "") + + Expect(k8sClient.Delete(ctx, inferenceService)).Should(Succeed()) + Consistently(func() int { + smmList := &maistrav1.ServiceMeshMemberList{} + Expect(k8sClient.List(ctx, smmList, client.InNamespace(inferenceService.Namespace))).Should(Succeed()) + return len(smmList.Items) + }).Should(Equal(1)) + }) + }) + + When("deleting a model, but there are other models left in the namespace", func() { + It("it should not remove the owned service mesh enrolment", func() { + inferenceService1 := createInferenceService(testNs, "") + createInferenceService(testNs, "secondary-isvc") + expectOwnedSmmCreated(inferenceService1.Namespace) + + Expect(k8sClient.Delete(ctx, inferenceService1)).Should(Succeed()) + Consistently(func() error { + smm := &maistrav1.ServiceMeshMember{} + key := types.NamespacedName{Name: constants.ServiceMeshMemberName, Namespace: inferenceService1.Namespace} + err := k8sClient.Get(ctx, key, smm) + return err + }, timeout, interval).Should(Succeed()) + }) + }) + }) + + Describe("Dashboard reconciler", func() { + var testNs string + + createServingRuntime := func(namespace, path string) *kservev1alpha1.ServingRuntime { + servingRuntime := &kservev1alpha1.ServingRuntime{} + err := testutils.ConvertToStructuredResource(path, servingRuntime) + Expect(err).NotTo(HaveOccurred()) + servingRuntime.SetNamespace(namespace) + if err := k8sClient.Create(ctx, servingRuntime); err != nil && !k8sErrors.IsAlreadyExists(err) { + Fail(err.Error()) + } + return servingRuntime + } + + createInferenceService := func(namespace, name string, path string) *kservev1beta1.InferenceService { + inferenceService := &kservev1beta1.InferenceService{} + err := testutils.ConvertToStructuredResource(path, inferenceService) + Expect(err).NotTo(HaveOccurred()) + inferenceService.SetNamespace(namespace) + if len(name) != 0 { + inferenceService.Name = name + } + if err := k8sClient.Create(ctx, inferenceService); err != nil && !k8sErrors.IsAlreadyExists(err) { + Fail(err.Error()) + } + return inferenceService + } + + BeforeEach(func() { + testNs = testutils.Namespaces.Create(ctx, k8sClient).Name + + inferenceServiceConfig := &corev1.ConfigMap{} + Expect(testutils.ConvertToStructuredResource(InferenceServiceConfigPath1, inferenceServiceConfig)).To(Succeed()) + if err := k8sClient.Create(ctx, inferenceServiceConfig); err != nil && !k8sErrors.IsAlreadyExists(err) { + Fail(err.Error()) + } + }) + + When("deploying a Kserve model", func() { + It("if the runtime is supported for metrics, it should create a configmap with prometheus queries", func() { + _ = createServingRuntime(testNs, KserveServingRuntimePath1) + _ = createInferenceService(testNs, KserveOvmsInferenceServiceName, KserveInferenceServicePath1) + + metricsConfigMap, err := testutils.WaitForConfigMap(k8sClient, testNs, KserveOvmsInferenceServiceName+constants.KserveMetricsConfigMapNameSuffix, 30, 1*time.Second) + Expect(err).NotTo(HaveOccurred()) + Expect(metricsConfigMap).NotTo(BeNil()) + + finaldata := utils.SubstituteVariablesInQueries(constants.OvmsMetricsData, testNs, KserveOvmsInferenceServiceName) + expectedMetricsConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: KserveOvmsInferenceServiceName + constants.KserveMetricsConfigMapNameSuffix, + Namespace: testNs, + }, + Data: map[string]string{ + "supported": "true", + "metrics": finaldata, + }, + } + Expect(testutils.CompareConfigMap(metricsConfigMap, expectedMetricsConfigMap)).Should(BeTrue()) + Expect(expectedMetricsConfigMap.Data).NotTo(HaveKeyWithValue("metrics", ContainSubstring("${REQUEST_RATE_INTERVAL}"))) + }) + + It("if the runtime is not supported for metrics, it should create a configmap with the unsupported config", func() { + _ = createServingRuntime(testNs, UnsupprtedMetricsServingRuntimePath) + _ = createInferenceService(testNs, UnsupportedMetricsInferenceServiceName, UnsupportedMetricsInferenceServicePath) + + metricsConfigMap, err := testutils.WaitForConfigMap(k8sClient, testNs, UnsupportedMetricsInferenceServiceName+constants.KserveMetricsConfigMapNameSuffix, 30, 1*time.Second) + Expect(err).NotTo(HaveOccurred()) + Expect(metricsConfigMap).NotTo(BeNil()) + + expectedMetricsConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: UnsupportedMetricsInferenceServiceName + constants.KserveMetricsConfigMapNameSuffix, + Namespace: testNs, + }, + Data: map[string]string{ + "supported": "false", + }, + } + Expect(testutils.CompareConfigMap(metricsConfigMap, expectedMetricsConfigMap)).Should(BeTrue()) + }) + + It("if the isvc does not have a runtime specified, an unsupported metrics configmap should be created", func() { + _ = createInferenceService(testNs, NilRuntimeInferenceServiceName, NilRuntimeInferenceServicePath) + + metricsConfigMap, err := testutils.WaitForConfigMap(k8sClient, testNs, NilRuntimeInferenceServiceName+constants.KserveMetricsConfigMapNameSuffix, 30, 1*time.Second) + Expect(err).NotTo(HaveOccurred()) + Expect(metricsConfigMap).NotTo(BeNil()) + + expectedmetricsConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: NilRuntimeInferenceServiceName + constants.KserveMetricsConfigMapNameSuffix, + Namespace: testNs, + }, + Data: map[string]string{ + "supported": "false", + }, + } + Expect(testutils.CompareConfigMap(metricsConfigMap, expectedmetricsConfigMap)).Should(BeTrue()) + }) + + It("if the isvc does not have the model field specified, an unsupported metrics configmap should be created", func() { + _ = createInferenceService(testNs, NilModelInferenceServiceName, NilModelInferenceServicePath) + + metricsConfigMap, err := testutils.WaitForConfigMap(k8sClient, testNs, NilModelInferenceServiceName+constants.KserveMetricsConfigMapNameSuffix, 30, 1*time.Second) + Expect(err).NotTo(HaveOccurred()) + Expect(metricsConfigMap).NotTo(BeNil()) + + expectedCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: NilModelInferenceServiceName + constants.KserveMetricsConfigMapNameSuffix, + Namespace: testNs, + }, + Data: map[string]string{ + "supported": "false", + }, + } + Expect(testutils.CompareConfigMap(metricsConfigMap, expectedCM)).Should(BeTrue()) + }) + }) + + When("deleting the deployed models", func() { + It("it should delete the associated configmap", func() { + _ = createServingRuntime(testNs, KserveServingRuntimePath1) + OvmsInferenceService := createInferenceService(testNs, KserveOvmsInferenceServiceName, KserveInferenceServicePath1) + + Expect(k8sClient.Delete(ctx, OvmsInferenceService)).Should(Succeed()) + Eventually(func() error { + configmap := &corev1.ConfigMap{} + key := types.NamespacedName{Name: KserveOvmsInferenceServiceName + constants.KserveMetricsConfigMapNameSuffix, Namespace: OvmsInferenceService.Namespace} + err := k8sClient.Get(ctx, key, configmap) + return err + }, timeout, interval).ShouldNot(Succeed()) + + _ = createServingRuntime(testNs, UnsupprtedMetricsServingRuntimePath) + SklearnInferenceService := createInferenceService(testNs, UnsupportedMetricsInferenceServiceName, UnsupportedMetricsInferenceServicePath) + + Expect(k8sClient.Delete(ctx, SklearnInferenceService)).Should(Succeed()) + Eventually(func() error { + configmap := &corev1.ConfigMap{} + key := types.NamespacedName{Name: UnsupportedMetricsInferenceServiceName + constants.KserveMetricsConfigMapNameSuffix, Namespace: SklearnInferenceService.Namespace} + err := k8sClient.Get(ctx, key, configmap) + return err + }, timeout, interval).ShouldNot(Succeed()) + }) + }) + }) + + Describe("InferenceService Authorization", func() { + var ( + namespace *corev1.Namespace + isvc *kservev1beta1.InferenceService + ) + + When("not configured for the cluster", func() { + BeforeEach(func() { + ctx := context.Background() + namespace = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testutils.Namespaces.Get(), + }, + } + Expect(k8sClient.Create(ctx, namespace)).Should(Succeed()) + inferenceServiceConfig := &corev1.ConfigMap{} + + Expect(testutils.ConvertToStructuredResource(InferenceServiceConfigPath1, inferenceServiceConfig)).To(Succeed()) + if err := k8sClient.Create(ctx, inferenceServiceConfig); err != nil && !k8sErrors.IsAlreadyExists(err) { + Fail(err.Error()) + } + + // We need to stub the cluster state and indicate if Authorino is configured as authorization layer + if dsciErr := createDSCI(DSCIWithoutAuthorization); dsciErr != nil && !k8sErrors.IsAlreadyExists(dsciErr) { + Fail(dsciErr.Error()) + } + + isvc = createISVCWithoutAuth(namespace.Name) + }) + + AfterEach(func() { + Expect(deleteDSCI(DSCIWithoutAuthorization)).To(Succeed()) + }) + + It("should not create auth config", func() { + Consistently(func() error { + ac := &authorinov1beta2.AuthConfig{} + return getAuthConfig(namespace.Name, isvc.Name, ac) + }). + WithTimeout(timeout). + WithPolling(interval). + Should(Not(Succeed())) + }) + }) + + When("configured for the cluster", func() { + + BeforeEach(func() { + ctx := context.Background() + namespace = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testutils.Namespaces.Get(), + }, + } + Expect(k8sClient.Create(ctx, namespace)).Should(Succeed()) + inferenceServiceConfig := &corev1.ConfigMap{} + + Expect(testutils.ConvertToStructuredResource(InferenceServiceConfigPath1, inferenceServiceConfig)).To(Succeed()) + if err := k8sClient.Create(ctx, inferenceServiceConfig); err != nil && !k8sErrors.IsAlreadyExists(err) { + Fail(err.Error()) + } + + // // We need to stub the cluster state and indicate that Authorino is configured as authorization layer + // if dsciErr := createDSCI(DSCIWithAuthorization); dsciErr != nil && !k8sErrors.IsAlreadyExists(dsciErr) { + // Fail(dsciErr.Error()) + // } + + // TODO: See utils.VerifyIfMeshAuthorizationIsEnabled func + if authPolicyErr := createAuthorizationPolicy(KServeAuthorizationPolicy); authPolicyErr != nil && !k8sErrors.IsAlreadyExists(authPolicyErr) { + Fail(authPolicyErr.Error()) + } + }) + + AfterEach(func() { + // Expect(deleteDSCI(DSCIWithAuthorization)).To(Succeed()) + Expect(deleteAuthorizationPolicy(KServeAuthorizationPolicy)).To(Succeed()) + }) + + Context("when InferenceService is not ready", func() { + BeforeEach(func() { + isvc = createISVCMissingStatus(namespace.Name) + }) + + It("should not create auth config on missing status.URL", func() { + + Consistently(func() error { + ac := &authorinov1beta2.AuthConfig{} + return getAuthConfig(namespace.Name, isvc.Name, ac) + }). + WithTimeout(timeout). + WithPolling(interval). + Should(Not(Succeed())) + }) + }) + + Context("when InferenceService is ready", func() { + + Context("auth not enabled", func() { + BeforeEach(func() { + isvc = createISVCWithoutAuth(namespace.Name) + }) + + It("should create anonymous auth config", func() { + Expect(updateISVCStatus(isvc)).To(Succeed()) + + Eventually(func(g Gomega) { + ac := &authorinov1beta2.AuthConfig{} + g.Expect(getAuthConfig(namespace.Name, isvc.Name, ac)).To(Succeed()) + g.Expect(ac.Spec.Authorization["anonymous-access"]).NotTo(BeNil()) + }). + WithTimeout(timeout). + WithPolling(interval). + Should(Succeed()) + }) + + It("should update to non anonymous on enable", func() { + Expect(updateISVCStatus(isvc)).To(Succeed()) + + Eventually(func(g Gomega) { + ac := &authorinov1beta2.AuthConfig{} + g.Expect(getAuthConfig(namespace.Name, isvc.Name, ac)).To(Succeed()) + g.Expect(ac.Spec.Authorization["anonymous-access"]).NotTo(BeNil()) + }). + WithTimeout(timeout). + WithPolling(interval). + Should(Succeed()) + + Expect(enableAuth(isvc)).To(Succeed()) + Eventually(func(g Gomega) { + ac := &authorinov1beta2.AuthConfig{} + g.Expect(ac.Spec.Authorization["kubernetes-user"]).NotTo(BeNil()) + g.Expect(getAuthConfig(namespace.Name, isvc.Name, ac)).To(Succeed()) + }). + WithTimeout(timeout). + WithPolling(interval). + Should(Succeed()) + }) + }) + + Context("auth enabled", func() { + BeforeEach(func() { + isvc = createISVCWithAuth(namespace.Name) + }) + + It("should create user defined auth config", func() { + Expect(updateISVCStatus(isvc)).To(Succeed()) + + Eventually(func(g Gomega) { + ac := &authorinov1beta2.AuthConfig{} + g.Expect(getAuthConfig(namespace.Name, isvc.Name, ac)).To(Succeed()) + g.Expect(ac.Spec.Authorization["kubernetes-user"]).NotTo(BeNil()) + }). + WithTimeout(timeout). + WithPolling(interval). + Should(Succeed()) + }) + + It("should update to anonymous on disable", func() { + Expect(updateISVCStatus(isvc)).To(Succeed()) + + Eventually(func(g Gomega) { + ac := &authorinov1beta2.AuthConfig{} + g.Expect(getAuthConfig(namespace.Name, isvc.Name, ac)).To(Succeed()) + g.Expect(ac.Spec.Authorization["kubernetes-user"]).NotTo(BeNil()) + }). + WithTimeout(timeout). + WithPolling(interval). + Should(Succeed()) + + Expect(disableAuth(isvc)).To(Succeed()) + Eventually(func(g Gomega) { + ac := &authorinov1beta2.AuthConfig{} + g.Expect(getAuthConfig(namespace.Name, isvc.Name, ac)).To(Succeed()) + g.Expect(ac.Spec.Authorization["anonymous-access"]).NotTo(BeNil()) + }). + WithTimeout(timeout). + WithPolling(interval). + Should(Succeed()) + }) + }) + }) + }) + + }) + + Describe("The KServe Raw reconciler", func() { + var testNs string + createServingRuntime := func(namespace, path string) *kservev1alpha1.ServingRuntime { + servingRuntime := &kservev1alpha1.ServingRuntime{} + err := testutils.ConvertToStructuredResource(path, servingRuntime) + Expect(err).NotTo(HaveOccurred()) + servingRuntime.SetNamespace(namespace) + if err := k8sClient.Create(ctx, servingRuntime); err != nil && !k8sErrors.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred()) + } + return servingRuntime + } + + createInferenceService := func(namespace, name string, path string) *kservev1beta1.InferenceService { + inferenceService := &kservev1beta1.InferenceService{} + err := testutils.ConvertToStructuredResource(path, inferenceService) + Expect(err).NotTo(HaveOccurred()) + inferenceService.SetNamespace(namespace) + if len(name) != 0 { + inferenceService.Name = name + } + inferenceService.Annotations = map[string]string{} + inferenceService.Annotations["serving.kserve.io/deploymentMode"] = "RawDeployment" + return inferenceService + } + + BeforeEach(func() { + testNs = testutils.Namespaces.Create(ctx, k8sClient).Name + + inferenceServiceConfig := &corev1.ConfigMap{} + Expect(testutils.ConvertToStructuredResource(InferenceServiceConfigPath1, inferenceServiceConfig)).To(Succeed()) + if err := k8sClient.Create(ctx, inferenceServiceConfig); err != nil && !k8sErrors.IsAlreadyExists(err) { + Fail(err.Error()) + } + + }) + + When("deploying a Kserve RawDeployment model", func() { + It("it should create a default clusterrolebinding for auth", func() { + _ = createServingRuntime(testNs, KserveServingRuntimePath1) + inferenceService := createInferenceService(testNs, KserveOvmsInferenceServiceName, KserveInferenceServicePath1) + if err := k8sClient.Create(ctx, inferenceService); err != nil && !k8sErrors.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred()) + } + + crb := &rbacv1.ClusterRoleBinding{} + Eventually(func() error { + key := types.NamespacedName{Name: inferenceService.Namespace + "-" + constants.KserveServiceAccountName + "-auth-delegator", + Namespace: inferenceService.Namespace} + return k8sClient.Get(ctx, key, crb) + }, timeout, interval).ShouldNot(HaveOccurred()) + + route := &routev1.Route{} + Eventually(func() error { + key := types.NamespacedName{Name: inferenceService.Name, Namespace: inferenceService.Namespace} + return k8sClient.Get(ctx, key, route) + }, timeout, interval).Should(HaveOccurred()) + }) + It("it should create a custom rolebinding if isvc has a SA defined", func() { + serviceAccountName := "custom-sa" + _ = createServingRuntime(testNs, KserveServingRuntimePath1) + inferenceService := createInferenceService(testNs, KserveOvmsInferenceServiceName, KserveInferenceServicePath1) + inferenceService.Spec.Predictor.ServiceAccountName = serviceAccountName + if err := k8sClient.Create(ctx, inferenceService); err != nil && !k8sErrors.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred()) + } + + crb := &rbacv1.ClusterRoleBinding{} + Eventually(func() error { + key := types.NamespacedName{Name: inferenceService.Namespace + "-" + serviceAccountName + "-auth-delegator", + Namespace: inferenceService.Namespace} + return k8sClient.Get(ctx, key, crb) + }, timeout, interval).ShouldNot(HaveOccurred()) + }) + It("it should create a route if isvc has the label to expose route", func() { + inferenceService := createInferenceService(testNs, KserveOvmsInferenceServiceName, KserveInferenceServicePath1) + inferenceService.Labels = map[string]string{} + inferenceService.Labels[constants.KserveNetworkVisibility] = constants.LabelEnableKserveRawRoute + // The service is manually created before the isvc otherwise the unit test risks running into a race condition + // where the reconcile loop finishes before the service is created, leading to no route being created. + isvcService := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: KserveOvmsInferenceServiceName + "-predictor", + Namespace: inferenceService.Namespace, + Annotations: map[string]string{ + "openshift.io/display-name": KserveOvmsInferenceServiceName, + "serving.kserve.io/deploymentMode": "RawDeployment", + }, + Labels: map[string]string{ + "app": "isvc." + KserveOvmsInferenceServiceName + "-predictor", + "component": "predictor", + "serving.kserve.io/inferenceservice": KserveOvmsInferenceServiceName, + }, + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "None", + IPFamilies: []corev1.IPFamily{"IPv4"}, + Ports: []corev1.ServicePort{ + { + Name: "http", + Protocol: corev1.ProtocolTCP, + Port: 8888, + TargetPort: intstr.FromString("http"), + }, + }, + ClusterIPs: []string{"None"}, + Selector: map[string]string{ + "app": "isvc." + KserveOvmsInferenceServiceName + "-predictor", + }, + }, + } + if err := k8sClient.Create(ctx, isvcService); err != nil && !k8sErrors.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred()) + } + service := &corev1.Service{} + Eventually(func() error { + key := types.NamespacedName{Name: isvcService.Name, Namespace: isvcService.Namespace} + return k8sClient.Get(ctx, key, service) + }, timeout, interval).Should(Succeed()) + + _ = createServingRuntime(testNs, KserveServingRuntimePath1) + if err := k8sClient.Create(ctx, inferenceService); err != nil && !k8sErrors.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred()) + } + + route := &routev1.Route{} + Eventually(func() error { + key := types.NamespacedName{Name: inferenceService.Name, Namespace: inferenceService.Namespace} + return k8sClient.Get(ctx, key, route) + }, timeout, interval).ShouldNot(HaveOccurred()) + }) + }) + When("deleting a Kserve RawDeployment model", func() { + It("the associated route should be deleted", func() { + _ = createServingRuntime(testNs, KserveServingRuntimePath1) + inferenceService := createInferenceService(testNs, KserveOvmsInferenceServiceName, KserveInferenceServicePath1) + if err := k8sClient.Create(ctx, inferenceService); err != nil && !k8sErrors.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred()) + } + + Expect(k8sClient.Delete(ctx, inferenceService)).Should(Succeed()) + + route := &routev1.Route{} + Eventually(func() error { + key := types.NamespacedName{Name: inferenceService.Name, Namespace: inferenceService.Namespace} + return k8sClient.Get(ctx, key, route) + }, timeout, interval).Should(HaveOccurred()) + }) + }) + When("namespace no longer has any RawDeployment models", func() { + It("should delete the default clusterrolebinding", func() { + _ = createServingRuntime(testNs, KserveServingRuntimePath1) + inferenceService := createInferenceService(testNs, KserveOvmsInferenceServiceName, KserveInferenceServicePath1) + if err := k8sClient.Create(ctx, inferenceService); err != nil && !k8sErrors.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred()) + } + Expect(k8sClient.Delete(ctx, inferenceService)).Should(Succeed()) + crb := &rbacv1.ClusterRoleBinding{} + Eventually(func() error { + namespacedNamed := types.NamespacedName{Name: testNs + "-" + constants.KserveServiceAccountName + "-auth-delegator", Namespace: WorkingNamespace} + err := k8sClient.Get(ctx, namespacedNamed, crb) + if k8sErrors.IsNotFound(err) { + return nil + } else { + return errors.New("crb deletion not detected") + } + }, timeout, interval).ShouldNot(HaveOccurred()) + }) + }) + }) +}) + +func withMatchingNestedField(path string, matcher gomegatypes.GomegaMatcher) gomegatypes.GomegaMatcher { + if path == "" { + Fail("cannot handle empty path") + } + + fields := strings.Split(path, ".") + + // Reverse the path, so we start composing matchers from the leaf up + for i, j := 0, len(fields)-1; i < j; i, j = i+1, j-1 { + fields[i], fields[j] = fields[j], fields[i] + } + + matchFields := MatchFields(IgnoreExtras, + Fields{fields[0]: matcher}, + ) + + for i := 1; i < len(fields); i++ { + matchFields = MatchFields(IgnoreExtras, Fields{fields[i]: matchFields}) + } + + return matchFields +} + +func getKServeRouteName(isvc *kservev1beta1.InferenceService) string { + return isvc.Name + "-" + isvc.Namespace +} + +func waitForUpdatedGatewayCompletion(cli client.Client, op string, namespace, gatewayName string, isvcName string) (*istioclientv1beta1.Gateway, error) { + ctx := context.Background() + portName := fmt.Sprintf("%s-%s", "https", isvcName) + gateway := &istioclientv1beta1.Gateway{} + + // Get the Gateway resource + err := cli.Get(ctx, client.ObjectKey{Namespace: namespace, Name: gatewayName}, gateway) + if err != nil { + return nil, fmt.Errorf("failed to get Gateway: %w", err) + } + + // Check conditions based on operation (op) + switch op { + case "add": + if !hasServerFromGateway(gateway, portName) { + return nil, fmt.Errorf("server %s not found in Gateway %s", portName, gatewayName) + } + case "delete": + if hasServerFromGateway(gateway, portName) { + return nil, fmt.Errorf("server %s still exists in Gateway %s", portName, gatewayName) + } + default: + return nil, fmt.Errorf("unsupported operation: %s", op) + } + + return gateway, nil +} + +// checks if the server exists for the given gateway +func hasServerFromGateway(gateway *istioclientv1beta1.Gateway, portName string) bool { + targetServerExist := false + for _, server := range gateway.Spec.Servers { + if server.Port.Name == portName { + targetServerExist = true + break + } + } + return targetServerExist +} + +func getAuthConfig(namespace, name string, ac *authorinov1beta2.AuthConfig) error { + return k8sClient.Get(context.Background(), types.NamespacedName{Namespace: namespace, Name: name}, ac) +} + +func createISVCMissingStatus(namespace string) *kservev1beta1.InferenceService { + inferenceService := &kservev1beta1.InferenceService{} + err := testutils.ConvertToStructuredResource(KserveInferenceServicePath1, inferenceService) + Expect(err).NotTo(HaveOccurred()) + inferenceService.Namespace = namespace + Expect(k8sClient.Create(ctx, inferenceService)).Should(Succeed()) + return inferenceService +} + +func createISVCWithAuth(namespace string) *kservev1beta1.InferenceService { + inferenceService := createBasicISVC(namespace) + inferenceService.Annotations[constants.LabelEnableAuth] = "true" + Expect(k8sClient.Create(ctx, inferenceService)).Should(Succeed()) + + return inferenceService +} + +func createISVCWithoutAuth(namespace string) *kservev1beta1.InferenceService { + inferenceService := createBasicISVC(namespace) + Expect(k8sClient.Create(ctx, inferenceService)).Should(Succeed()) + + return inferenceService +} + +func createBasicISVC(namespace string) *kservev1beta1.InferenceService { + inferenceService := &kservev1beta1.InferenceService{} + err := testutils.ConvertToStructuredResource(KserveInferenceServicePath1, inferenceService) + Expect(err).NotTo(HaveOccurred()) + inferenceService.Namespace = namespace + if inferenceService.Annotations == nil { + inferenceService.Annotations = map[string]string{} + } + return inferenceService +} + +func updateISVCStatus(isvc *kservev1beta1.InferenceService) error { + url, _ := apis.ParseURL("http://iscv-" + isvc.Namespace + "ns.apps.openshift.ai") + isvc.Status = kservev1beta1.InferenceServiceStatus{ + URL: url, + } + return k8sClient.Status().Update(context.Background(), isvc) +} + +func disableAuth(isvc *kservev1beta1.InferenceService) error { + delete(isvc.Annotations, constants.LabelEnableAuth) + delete(isvc.Annotations, constants.LabelEnableAuthODH) + return k8sClient.Update(context.Background(), isvc) +} + +func enableAuth(isvc *kservev1beta1.InferenceService) error { + if isvc.Annotations == nil { + isvc.Annotations = map[string]string{} + } + isvc.Annotations[constants.LabelEnableAuthODH] = "true" + return k8sClient.Update(context.Background(), isvc) +} + +func createDSCI(_ string) error { + dsci := DSCIWithoutAuthorization + obj := &unstructured.Unstructured{} + if err := testutils.ConvertToUnstructuredResource(dsci, obj); err != nil { + return err + } + + gvk := utils.GVK.DataScienceClusterInitialization + obj.SetGroupVersionKind(gvk) + dynamicClient, err := dynamic.NewForConfig(testEnv.Config) + if err != nil { + return err + } + + gvr := schema.GroupVersionResource{ + Group: gvk.Group, + Version: gvk.Version, + Resource: "dscinitializations", + } + resource := dynamicClient.Resource(gvr) + createdObj, createErr := resource.Create(context.TODO(), obj, metav1.CreateOptions{}) + if createErr != nil { + return nil + } + + if status, found, err := unstructured.NestedFieldCopy(obj.Object, "status"); err != nil { + return err + } else if found { + if err := unstructured.SetNestedField(createdObj.Object, status, "status"); err != nil { + return err + } + } + + _, statusErr := resource.UpdateStatus(context.TODO(), createdObj, metav1.UpdateOptions{}) + + return statusErr +} + +func createAuthorizationPolicy(authPolicyFile string) error { + obj := &unstructured.Unstructured{} + if err := testutils.ConvertToUnstructuredResource(authPolicyFile, obj); err != nil { + return err + } + + obj.SetGroupVersionKind(istiosecv1b1.SchemeGroupVersion.WithKind("AuthorizationPolicy")) + dynamicClient, err := dynamic.NewForConfig(testEnv.Config) + if err != nil { + return err + } + + gvr := istiosecv1b1.SchemeGroupVersion.WithResource("authorizationpolicies") + resource := dynamicClient.Resource(gvr) + _, meshNamespace := utils.GetIstioControlPlaneName(ctx, k8sClient) + _, createErr := resource.Namespace(meshNamespace).Create(context.TODO(), obj, metav1.CreateOptions{}) + + return createErr +} + +func deleteDSCI(dsci string) error { + obj := &unstructured.Unstructured{} + if err := testutils.ConvertToUnstructuredResource(dsci, obj); err != nil { + return err + } + + gvk := utils.GVK.DataScienceClusterInitialization + obj.SetGroupVersionKind(gvk) + dynamicClient, err := dynamic.NewForConfig(testEnv.Config) + if err != nil { + return err + } + + gvr := schema.GroupVersionResource{ + Group: gvk.Group, + Version: gvk.Version, + Resource: "dscinitializations", + } + return dynamicClient.Resource(gvr).Delete(context.TODO(), obj.GetName(), metav1.DeleteOptions{}) +} + +func deleteAuthorizationPolicy(authPolicyFile string) error { + obj := &unstructured.Unstructured{} + if err := testutils.ConvertToUnstructuredResource(authPolicyFile, obj); err != nil { + return err + } + + obj.SetGroupVersionKind(istiosecv1b1.SchemeGroupVersion.WithKind("AuthorizationPolicy")) + dynamicClient, err := dynamic.NewForConfig(testEnv.Config) + if err != nil { + return err + } + + gvr := istiosecv1b1.SchemeGroupVersion.WithResource("authorizationpolicies") + _, meshNamespace := utils.GetIstioControlPlaneName(ctx, k8sClient) + return dynamicClient.Resource(gvr).Namespace(meshNamespace).Delete(context.TODO(), obj.GetName(), metav1.DeleteOptions{}) +} diff --git a/controllers/reconcilers/kserve_authconfig_reconciler.go b/internal/controller/serving/reconcilers/kserve_authconfig_reconciler.go similarity index 92% rename from controllers/reconcilers/kserve_authconfig_reconciler.go rename to internal/controller/serving/reconcilers/kserve_authconfig_reconciler.go index 515b4eb3..819d78b7 100644 --- a/controllers/reconcilers/kserve_authconfig_reconciler.go +++ b/internal/controller/serving/reconcilers/kserve_authconfig_reconciler.go @@ -19,19 +19,20 @@ import ( "context" "fmt" - "github.com/opendatahub-io/odh-model-controller/controllers/utils" + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" "github.com/go-logr/logr" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" authorinov1beta2 "github.com/kuadrant/authorino/api/v1beta2" - "github.com/opendatahub-io/odh-model-controller/controllers/comparators" - "github.com/opendatahub-io/odh-model-controller/controllers/constants" - "github.com/opendatahub-io/odh-model-controller/controllers/processors" - "github.com/opendatahub-io/odh-model-controller/controllers/resources" k8serror "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/comparators" + "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" + "github.com/opendatahub-io/odh-model-controller/internal/controller/processors" + "github.com/opendatahub-io/odh-model-controller/internal/controller/resources" ) var _ SubResourceReconciler = (*KserveAuthConfigReconciler)(nil) @@ -127,7 +128,10 @@ func (r *KserveAuthConfigReconciler) createDesiredResource(ctx context.Context, } template.Labels[constants.LabelAuthGroup] = "default" - ctrl.SetControllerReference(isvc, &template, r.client.Scheme()) + err = ctrl.SetControllerReference(isvc, &template, r.client.Scheme()) + if err != nil { + return nil, err + } return &template, nil } diff --git a/controllers/reconcilers/kserve_istio_peerauthentication_reconciler.go b/internal/controller/serving/reconcilers/kserve_istio_peerauthentication_reconciler.go similarity index 93% rename from controllers/reconcilers/kserve_istio_peerauthentication_reconciler.go rename to internal/controller/serving/reconcilers/kserve_istio_peerauthentication_reconciler.go index 0ab851da..ec0f548f 100644 --- a/controllers/reconcilers/kserve_istio_peerauthentication_reconciler.go +++ b/internal/controller/serving/reconcilers/kserve_istio_peerauthentication_reconciler.go @@ -17,14 +17,16 @@ package reconcilers import ( "context" + "github.com/go-logr/logr" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - "github.com/opendatahub-io/odh-model-controller/controllers/comparators" - "github.com/opendatahub-io/odh-model-controller/controllers/processors" - "github.com/opendatahub-io/odh-model-controller/controllers/resources" istiosecv1beta1 "istio.io/client-go/pkg/apis/security/v1beta1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/comparators" + "github.com/opendatahub-io/odh-model-controller/internal/controller/processors" + "github.com/opendatahub-io/odh-model-controller/internal/controller/resources" ) const ( @@ -95,7 +97,7 @@ func (r *KserveIstioPeerAuthenticationReconciler) processDelta(ctx context.Conte if delta.IsAdded() { log.V(1).Info("Delta found", "create", desiredPeerAuthentication.GetName()) if err = r.client.Create(ctx, desiredPeerAuthentication); err != nil { - return + return err } } if delta.IsUpdated() { @@ -106,13 +108,13 @@ func (r *KserveIstioPeerAuthenticationReconciler) processDelta(ctx context.Conte rp.Spec.PortLevelMtls = desiredPeerAuthentication.Spec.PortLevelMtls if err = r.client.Update(ctx, rp); err != nil { - return + return err } } if delta.IsRemoved() { log.V(1).Info("Delta found", "delete", existingPeerAuthentication.GetName()) if err = r.client.Delete(ctx, existingPeerAuthentication); err != nil { - return + return err } } return nil diff --git a/controllers/reconcilers/kserve_istio_podmonitor_reconciler.go b/internal/controller/serving/reconcilers/kserve_istio_podmonitor_reconciler.go similarity index 91% rename from controllers/reconcilers/kserve_istio_podmonitor_reconciler.go rename to internal/controller/serving/reconcilers/kserve_istio_podmonitor_reconciler.go index bbcd3ac4..ab90ae07 100644 --- a/controllers/reconcilers/kserve_istio_podmonitor_reconciler.go +++ b/internal/controller/serving/reconcilers/kserve_istio_podmonitor_reconciler.go @@ -18,16 +18,18 @@ package reconcilers import ( "context" "fmt" + "github.com/go-logr/logr" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - "github.com/opendatahub-io/odh-model-controller/controllers/comparators" - "github.com/opendatahub-io/odh-model-controller/controllers/processors" - "github.com/opendatahub-io/odh-model-controller/controllers/resources" - "github.com/opendatahub-io/odh-model-controller/controllers/utils" v1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/comparators" + "github.com/opendatahub-io/odh-model-controller/internal/controller/processors" + "github.com/opendatahub-io/odh-model-controller/internal/controller/resources" + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" ) const ( @@ -55,10 +57,7 @@ func (r *KserveIstioPodMonitorReconciler) Reconcile(ctx context.Context, log log log.V(1).Info("Creating Istio PodMonitor for target namespace") // Create Desired resource - desiredResource, err := r.createDesiredResource(ctx, isvc) - if err != nil { - return err - } + desiredResource := r.createDesiredResource(ctx, isvc) // Get Existing resource existingResource, err := r.getExistingResource(ctx, log, isvc) @@ -78,7 +77,7 @@ func (r *KserveIstioPodMonitorReconciler) Cleanup(ctx context.Context, log logr. return r.podMonitorHandler.DeletePodMonitor(ctx, types.NamespacedName{Name: istioPodMonitorName, Namespace: isvcNs}) } -func (r *KserveIstioPodMonitorReconciler) createDesiredResource(ctx context.Context, isvc *kservev1beta1.InferenceService) (*v1.PodMonitor, error) { +func (r *KserveIstioPodMonitorReconciler) createDesiredResource(ctx context.Context, isvc *kservev1beta1.InferenceService) *v1.PodMonitor { istioControlPlaneName, meshNamespace := utils.GetIstioControlPlaneName(ctx, r.client) desiredPodMonitor := &v1.PodMonitor{ @@ -148,7 +147,7 @@ func (r *KserveIstioPodMonitorReconciler) createDesiredResource(ctx context.Cont }, }, } - return desiredPodMonitor, nil + return desiredPodMonitor } func (r *KserveIstioPodMonitorReconciler) getExistingResource(ctx context.Context, log logr.Logger, isvc *kservev1beta1.InferenceService) (*v1.PodMonitor, error) { @@ -167,7 +166,7 @@ func (r *KserveIstioPodMonitorReconciler) processDelta(ctx context.Context, log if delta.IsAdded() { log.V(1).Info("Delta found", "create", desiredPodMonitor.GetName()) if err = r.client.Create(ctx, desiredPodMonitor); err != nil { - return + return err } } if delta.IsUpdated() { @@ -176,13 +175,13 @@ func (r *KserveIstioPodMonitorReconciler) processDelta(ctx context.Context, log rp.Spec = desiredPodMonitor.Spec if err = r.client.Update(ctx, rp); err != nil { - return + return err } } if delta.IsRemoved() { log.V(1).Info("Delta found", "delete", existingPodMonitor.GetName()) if err = r.client.Delete(ctx, existingPodMonitor); err != nil { - return + return err } } return nil diff --git a/controllers/reconcilers/kserve_istio_servicemonitor_reconciler.go b/internal/controller/serving/reconcilers/kserve_istio_servicemonitor_reconciler.go similarity index 93% rename from controllers/reconcilers/kserve_istio_servicemonitor_reconciler.go rename to internal/controller/serving/reconcilers/kserve_istio_servicemonitor_reconciler.go index c484d33a..52ffd266 100644 --- a/controllers/reconcilers/kserve_istio_servicemonitor_reconciler.go +++ b/internal/controller/serving/reconcilers/kserve_istio_servicemonitor_reconciler.go @@ -17,14 +17,16 @@ package reconcilers import ( "context" + "github.com/go-logr/logr" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - "github.com/opendatahub-io/odh-model-controller/controllers/comparators" - "github.com/opendatahub-io/odh-model-controller/controllers/processors" - "github.com/opendatahub-io/odh-model-controller/controllers/resources" v1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/comparators" + "github.com/opendatahub-io/odh-model-controller/internal/controller/processors" + "github.com/opendatahub-io/odh-model-controller/internal/controller/resources" ) const ( @@ -96,7 +98,7 @@ func (r *KserveIstioServiceMonitorReconciler) processDelta(ctx context.Context, if delta.IsAdded() { log.V(1).Info("Delta found", "create", desiredServiceMonitor.GetName()) if err = r.client.Create(ctx, desiredServiceMonitor); err != nil { - return + return err } } if delta.IsUpdated() { @@ -107,13 +109,13 @@ func (r *KserveIstioServiceMonitorReconciler) processDelta(ctx context.Context, rp.Spec = desiredServiceMonitor.Spec if err = r.client.Update(ctx, rp); err != nil { - return + return err } } if delta.IsRemoved() { log.V(1).Info("Delta found", "delete", existingServiceMonitor.GetName()) if err = r.client.Delete(ctx, existingServiceMonitor); err != nil { - return + return err } } return nil diff --git a/controllers/reconcilers/kserve_istio_smm_reconciler.go b/internal/controller/serving/reconcilers/kserve_istio_smm_reconciler.go similarity index 89% rename from controllers/reconcilers/kserve_istio_smm_reconciler.go rename to internal/controller/serving/reconcilers/kserve_istio_smm_reconciler.go index 63274bb0..ac7c3ce7 100644 --- a/controllers/reconcilers/kserve_istio_smm_reconciler.go +++ b/internal/controller/serving/reconcilers/kserve_istio_smm_reconciler.go @@ -10,11 +10,11 @@ import ( v1 "maistra.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/opendatahub-io/odh-model-controller/controllers/comparators" - "github.com/opendatahub-io/odh-model-controller/controllers/constants" - "github.com/opendatahub-io/odh-model-controller/controllers/processors" - "github.com/opendatahub-io/odh-model-controller/controllers/resources" - "github.com/opendatahub-io/odh-model-controller/controllers/utils" + "github.com/opendatahub-io/odh-model-controller/internal/controller/comparators" + "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" + "github.com/opendatahub-io/odh-model-controller/internal/controller/processors" + "github.com/opendatahub-io/odh-model-controller/internal/controller/resources" + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" ) type KserveServiceMeshMemberReconciler struct { @@ -36,10 +36,7 @@ func (r *KserveServiceMeshMemberReconciler) Reconcile(ctx context.Context, log l log.V(1).Info("Verifying that the namespace is enrolled to the mesh") // Create Desired resource - desiredResource, err := r.createDesiredResource(ctx, isvc) - if err != nil { - return err - } + desiredResource := r.createDesiredResource(ctx, isvc) // Get Existing resource existingResource, err := r.getExistingResource(ctx, log, isvc.Namespace) @@ -63,7 +60,7 @@ func (r *KserveServiceMeshMemberReconciler) Cleanup(ctx context.Context, log log return r.processDelta(ctx, log, nil, existingSMM) } -func (r *KserveServiceMeshMemberReconciler) createDesiredResource(ctx context.Context, isvc *kservev1beta1.InferenceService) (*v1.ServiceMeshMember, error) { +func (r *KserveServiceMeshMemberReconciler) createDesiredResource(ctx context.Context, isvc *kservev1beta1.InferenceService) *v1.ServiceMeshMember { smcpName, smcpNamespace := utils.GetIstioControlPlaneName(ctx, r.client) return &v1.ServiceMeshMember{ @@ -84,7 +81,7 @@ func (r *KserveServiceMeshMemberReconciler) createDesiredResource(ctx context.Co Namespace: smcpNamespace, }, }, - }, nil + } } func (r *KserveServiceMeshMemberReconciler) getExistingResource(ctx context.Context, log logr.Logger, namespace string) (*v1.ServiceMeshMember, error) { diff --git a/controllers/reconcilers/kserve_istio_telemetry_reconciler.go b/internal/controller/serving/reconcilers/kserve_istio_telemetry_reconciler.go similarity index 90% rename from controllers/reconcilers/kserve_istio_telemetry_reconciler.go rename to internal/controller/serving/reconcilers/kserve_istio_telemetry_reconciler.go index 9652a11e..36a09942 100644 --- a/controllers/reconcilers/kserve_istio_telemetry_reconciler.go +++ b/internal/controller/serving/reconcilers/kserve_istio_telemetry_reconciler.go @@ -17,17 +17,19 @@ package reconcilers import ( "context" + "github.com/go-logr/logr" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - "github.com/opendatahub-io/odh-model-controller/controllers/comparators" - "github.com/opendatahub-io/odh-model-controller/controllers/processors" - "github.com/opendatahub-io/odh-model-controller/controllers/resources" "istio.io/api/telemetry/v1alpha1" istiotypes "istio.io/api/type/v1beta1" telemetryv1alpha1 "istio.io/client-go/pkg/apis/telemetry/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/comparators" + "github.com/opendatahub-io/odh-model-controller/internal/controller/processors" + "github.com/opendatahub-io/odh-model-controller/internal/controller/resources" ) const ( @@ -55,10 +57,7 @@ func (r *KserveIstioTelemetryReconciler) Reconcile(ctx context.Context, log logr log.V(1).Info("Creating Istio Telemetry object for target namespace") // Create Desired resource - desiredResource, err := r.createDesiredResource(isvc) - if err != nil { - return err - } + desiredResource := r.createDesiredResource(isvc) // Get Existing resource existingResource, err := r.getExistingResource(ctx, log, isvc) @@ -78,7 +77,7 @@ func (r *KserveIstioTelemetryReconciler) Cleanup(ctx context.Context, log logr.L return r.telemetryHandler.DeleteTelemetry(ctx, types.NamespacedName{Name: telemetryName, Namespace: isvcNs}) } -func (r *KserveIstioTelemetryReconciler) createDesiredResource(isvc *kservev1beta1.InferenceService) (*telemetryv1alpha1.Telemetry, error) { +func (r *KserveIstioTelemetryReconciler) createDesiredResource(isvc *kservev1beta1.InferenceService) *telemetryv1alpha1.Telemetry { desiredTelemetry := &telemetryv1alpha1.Telemetry{ ObjectMeta: metav1.ObjectMeta{ Name: telemetryName, @@ -101,7 +100,7 @@ func (r *KserveIstioTelemetryReconciler) createDesiredResource(isvc *kservev1bet }, }, } - return desiredTelemetry, nil + return desiredTelemetry } func (r *KserveIstioTelemetryReconciler) getExistingResource(ctx context.Context, log logr.Logger, isvc *kservev1beta1.InferenceService) (*telemetryv1alpha1.Telemetry, error) { @@ -120,7 +119,7 @@ func (r *KserveIstioTelemetryReconciler) processDelta(ctx context.Context, log l if delta.IsAdded() { log.V(1).Info("Delta found", "create", desiredTelemetry.GetName()) if err = r.client.Create(ctx, desiredTelemetry); err != nil { - return + return err } } if delta.IsUpdated() { @@ -130,13 +129,13 @@ func (r *KserveIstioTelemetryReconciler) processDelta(ctx context.Context, log l rp.Spec.Metrics = desiredTelemetry.Spec.Metrics if err = r.client.Update(ctx, rp); err != nil { - return + return err } } if delta.IsRemoved() { log.V(1).Info("Delta found", "delete", existingTelemetry.GetName()) if err = r.client.Delete(ctx, existingTelemetry); err != nil { - return + return err } } return nil diff --git a/controllers/reconcilers/kserve_isvc_gateway_reconciler.go b/internal/controller/serving/reconcilers/kserve_isvc_gateway_reconciler.go similarity index 96% rename from controllers/reconcilers/kserve_isvc_gateway_reconciler.go rename to internal/controller/serving/reconcilers/kserve_isvc_gateway_reconciler.go index 909deeda..37aa0a30 100644 --- a/controllers/reconcilers/kserve_isvc_gateway_reconciler.go +++ b/internal/controller/serving/reconcilers/kserve_isvc_gateway_reconciler.go @@ -23,11 +23,11 @@ import ( "github.com/go-logr/logr" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - "github.com/opendatahub-io/odh-model-controller/controllers/comparators" - "github.com/opendatahub-io/odh-model-controller/controllers/constants" - "github.com/opendatahub-io/odh-model-controller/controllers/processors" - "github.com/opendatahub-io/odh-model-controller/controllers/resources" - "github.com/opendatahub-io/odh-model-controller/controllers/utils" + "github.com/opendatahub-io/odh-model-controller/internal/controller/comparators" + "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" + "github.com/opendatahub-io/odh-model-controller/internal/controller/processors" + "github.com/opendatahub-io/odh-model-controller/internal/controller/resources" + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/controllers/reconcilers/kserve_isvc_service_cert_reconciler.go b/internal/controller/serving/reconcilers/kserve_isvc_service_cert_reconciler.go similarity index 93% rename from controllers/reconcilers/kserve_isvc_service_cert_reconciler.go rename to internal/controller/serving/reconcilers/kserve_isvc_service_cert_reconciler.go index ec083ca7..e107b0d9 100644 --- a/controllers/reconcilers/kserve_isvc_service_cert_reconciler.go +++ b/internal/controller/serving/reconcilers/kserve_isvc_service_cert_reconciler.go @@ -20,12 +20,13 @@ import ( "github.com/go-logr/logr" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - "github.com/opendatahub-io/odh-model-controller/controllers/constants" - "github.com/opendatahub-io/odh-model-controller/controllers/resources" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" + "github.com/opendatahub-io/odh-model-controller/internal/controller/resources" ) var _ SubResourceReconciler = (*KserveIsvcServiceReconciler)(nil) @@ -63,10 +64,7 @@ func (r *KserveIsvcServiceReconciler) Reconcile(ctx context.Context, log logr.Lo } // Create Desired resource - desiredResource, err := r.createDesiredResource(isvc) - if err != nil { - return err - } + desiredResource := r.createDesiredResource(isvc) // Get Existing resource existingResource, err := r.getExistingResource(ctx, log, isvc) @@ -81,7 +79,7 @@ func (r *KserveIsvcServiceReconciler) Reconcile(ctx context.Context, log logr.Lo return nil } -func (r *KserveIsvcServiceReconciler) createDesiredResource(isvc *kservev1beta1.InferenceService) (*v1.Service, error) { +func (r *KserveIsvcServiceReconciler) createDesiredResource(isvc *kservev1beta1.InferenceService) *v1.Service { service := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "isvc-service", @@ -90,7 +88,7 @@ func (r *KserveIsvcServiceReconciler) createDesiredResource(isvc *kservev1beta1. }, }, } - return service, nil + return service } func (r *KserveIsvcServiceReconciler) getExistingResource(ctx context.Context, log logr.Logger, isvc *kservev1beta1.InferenceService) (*v1.Service, error) { diff --git a/controllers/reconcilers/kserve_metrics_dashboard_reconciler.go b/internal/controller/serving/reconcilers/kserve_metrics_dashboard_reconciler.go similarity index 93% rename from controllers/reconcilers/kserve_metrics_dashboard_reconciler.go rename to internal/controller/serving/reconcilers/kserve_metrics_dashboard_reconciler.go index bf3d3ead..1e4702ca 100644 --- a/controllers/reconcilers/kserve_metrics_dashboard_reconciler.go +++ b/internal/controller/serving/reconcilers/kserve_metrics_dashboard_reconciler.go @@ -17,18 +17,20 @@ package reconcilers import ( "context" + "regexp" + "strconv" + "github.com/go-logr/logr" "github.com/hashicorp/errwrap" kservev1alpha1 "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - "github.com/opendatahub-io/odh-model-controller/controllers/comparators" - "github.com/opendatahub-io/odh-model-controller/controllers/constants" - "github.com/opendatahub-io/odh-model-controller/controllers/processors" - "github.com/opendatahub-io/odh-model-controller/controllers/resources" - "github.com/opendatahub-io/odh-model-controller/controllers/utils" - "regexp" ctrl "sigs.k8s.io/controller-runtime" - "strconv" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/comparators" + "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" + "github.com/opendatahub-io/odh-model-controller/internal/controller/processors" + "github.com/opendatahub-io/odh-model-controller/internal/controller/resources" + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -96,7 +98,7 @@ func (r *KserveMetricsDashboardReconciler) createDesiredResource(ctx context.Con supported := false // there is the possibility to also have and nil model field, e.g: - //predictor: + // predictor: // containers: // - name: kserve-container // image: user/custom-model:v1 @@ -124,7 +126,7 @@ func (r *KserveMetricsDashboardReconciler) createDesiredResource(ctx context.Con if (runtime.Spec.Containers == nil) || (len(runtime.Spec.Containers) < 1) { log.V(1).Info("Could not determine runtime image") - supported = false + // supported = false } // supported is true only when a match on this map is found, is false otherwise @@ -182,7 +184,7 @@ func (r *KserveMetricsDashboardReconciler) processDelta(ctx context.Context, log if delta.IsAdded() { log.V(1).Info("Delta found", "create", desiredResource.GetName()) if err = r.client.Create(ctx, desiredResource); err != nil { - return + return err } } if delta.IsUpdated() { @@ -192,13 +194,13 @@ func (r *KserveMetricsDashboardReconciler) processDelta(ctx context.Context, log rp.Data = desiredResource.Data if err = r.client.Update(ctx, rp); err != nil { - return + return err } } if delta.IsRemoved() { log.V(1).Info("Delta found", "delete", existingResource.GetName()) if err = r.client.Delete(ctx, existingResource); err != nil { - return + return err } } return nil diff --git a/controllers/reconcilers/kserve_metrics_service_reconciler.go b/internal/controller/serving/reconcilers/kserve_metrics_service_reconciler.go similarity index 91% rename from controllers/reconcilers/kserve_metrics_service_reconciler.go rename to internal/controller/serving/reconcilers/kserve_metrics_service_reconciler.go index a006263c..f80f4287 100644 --- a/controllers/reconcilers/kserve_metrics_service_reconciler.go +++ b/internal/controller/serving/reconcilers/kserve_metrics_service_reconciler.go @@ -17,18 +17,16 @@ package reconcilers import ( "context" + "github.com/go-logr/logr" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - "github.com/opendatahub-io/odh-model-controller/controllers/comparators" - "github.com/opendatahub-io/odh-model-controller/controllers/processors" - "github.com/opendatahub-io/odh-model-controller/controllers/resources" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" -) -const ( - inferenceServiceLabelName = "serving.kserve.io/inferenceservice" + "github.com/opendatahub-io/odh-model-controller/internal/controller/comparators" + "github.com/opendatahub-io/odh-model-controller/internal/controller/processors" + "github.com/opendatahub-io/odh-model-controller/internal/controller/resources" ) var _ SubResourceReconciler = (*KserveMetricsServiceReconciler)(nil) @@ -91,7 +89,7 @@ func (r *KserveMetricsServiceReconciler) processDelta(ctx context.Context, log l if delta.IsAdded() { log.V(1).Info("Delta found", "create", desiredService.GetName()) if err = r.client.Create(ctx, desiredService); err != nil { - return + return err } } if delta.IsUpdated() { @@ -102,13 +100,13 @@ func (r *KserveMetricsServiceReconciler) processDelta(ctx context.Context, log l rp.Spec = desiredService.Spec if err = r.client.Update(ctx, rp); err != nil { - return + return err } } if delta.IsRemoved() { log.V(1).Info("Delta found", "delete", existingService.GetName()) if err = r.client.Delete(ctx, existingService); err != nil { - return + return err } } return nil diff --git a/controllers/reconcilers/kserve_metrics_servicemonitor_reconciler.go b/internal/controller/serving/reconcilers/kserve_metrics_servicemonitor_reconciler.go similarity index 92% rename from controllers/reconcilers/kserve_metrics_servicemonitor_reconciler.go rename to internal/controller/serving/reconcilers/kserve_metrics_servicemonitor_reconciler.go index 9350eaa2..7681f723 100644 --- a/controllers/reconcilers/kserve_metrics_servicemonitor_reconciler.go +++ b/internal/controller/serving/reconcilers/kserve_metrics_servicemonitor_reconciler.go @@ -17,14 +17,16 @@ package reconcilers import ( "context" + "github.com/go-logr/logr" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - "github.com/opendatahub-io/odh-model-controller/controllers/comparators" - "github.com/opendatahub-io/odh-model-controller/controllers/processors" - "github.com/opendatahub-io/odh-model-controller/controllers/resources" v1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/comparators" + "github.com/opendatahub-io/odh-model-controller/internal/controller/processors" + "github.com/opendatahub-io/odh-model-controller/internal/controller/resources" ) var _ SubResourceReconciler = (*KserveMetricsServiceMonitorReconciler)(nil) @@ -88,7 +90,7 @@ func (r *KserveMetricsServiceMonitorReconciler) processDelta(ctx context.Context if delta.IsAdded() { log.V(1).Info("Delta found", "create", desiredServiceMonitor.GetName()) if err = r.client.Create(ctx, desiredServiceMonitor); err != nil { - return + return err } } if delta.IsUpdated() { @@ -97,13 +99,13 @@ func (r *KserveMetricsServiceMonitorReconciler) processDelta(ctx context.Context rp.Spec = desiredServiceMonitor.Spec if err = r.client.Update(ctx, rp); err != nil { - return + return err } } if delta.IsRemoved() { log.V(1).Info("Delta found", "delete", existingServiceMonitor.GetName()) if err = r.client.Delete(ctx, existingServiceMonitor); err != nil { - return + return err } } return nil diff --git a/controllers/reconcilers/kserve_networkpolicy_reconciler.go b/internal/controller/serving/reconcilers/kserve_networkpolicy_reconciler.go similarity index 96% rename from controllers/reconcilers/kserve_networkpolicy_reconciler.go rename to internal/controller/serving/reconcilers/kserve_networkpolicy_reconciler.go index 397a36cb..7853527e 100644 --- a/controllers/reconcilers/kserve_networkpolicy_reconciler.go +++ b/internal/controller/serving/reconcilers/kserve_networkpolicy_reconciler.go @@ -17,16 +17,18 @@ package reconcilers import ( "context" + "github.com/go-logr/logr" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" "github.com/kserve/kserve/pkg/constants" - "github.com/opendatahub-io/odh-model-controller/controllers/comparators" - "github.com/opendatahub-io/odh-model-controller/controllers/processors" - "github.com/opendatahub-io/odh-model-controller/controllers/resources" - "k8s.io/api/networking/v1" + v1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/comparators" + "github.com/opendatahub-io/odh-model-controller/internal/controller/processors" + "github.com/opendatahub-io/odh-model-controller/internal/controller/resources" ) const ( @@ -211,7 +213,7 @@ func (r *KserveNetworkPolicyReconciler) processDelta(ctx context.Context, log lo if delta.IsAdded() { log.V(1).Info("Delta found", "create", desiredNetworkPolicy.GetName()) if err = r.client.Create(ctx, desiredNetworkPolicy); err != nil { - return + return err } } if delta.IsUpdated() { @@ -221,13 +223,13 @@ func (r *KserveNetworkPolicyReconciler) processDelta(ctx context.Context, log lo rp.Spec = desiredNetworkPolicy.Spec if err = r.client.Update(ctx, rp); err != nil { - return + return err } } if delta.IsRemoved() { log.V(1).Info("Delta found", "delete", existingNetworkPolicy.GetName()) if err = r.client.Delete(ctx, existingNetworkPolicy); err != nil { - return + return err } } return nil diff --git a/controllers/reconcilers/kserve_prometheus_rolebinding_reconciler.go b/internal/controller/serving/reconcilers/kserve_prometheus_rolebinding_reconciler.go similarity index 90% rename from controllers/reconcilers/kserve_prometheus_rolebinding_reconciler.go rename to internal/controller/serving/reconcilers/kserve_prometheus_rolebinding_reconciler.go index b65fc4ad..f2133ac3 100644 --- a/controllers/reconcilers/kserve_prometheus_rolebinding_reconciler.go +++ b/internal/controller/serving/reconcilers/kserve_prometheus_rolebinding_reconciler.go @@ -17,15 +17,17 @@ package reconcilers import ( "context" + "github.com/go-logr/logr" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - "github.com/opendatahub-io/odh-model-controller/controllers/comparators" - "github.com/opendatahub-io/odh-model-controller/controllers/processors" - "github.com/opendatahub-io/odh-model-controller/controllers/resources" v1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/comparators" + "github.com/opendatahub-io/odh-model-controller/internal/controller/processors" + "github.com/opendatahub-io/odh-model-controller/internal/controller/resources" ) const ( @@ -53,10 +55,7 @@ func (r *KservePrometheusRoleBindingReconciler) Reconcile(ctx context.Context, l log.V(1).Info("Verifying that the role binding to enable prometheus access exists") // Create Desired resource - desiredResource, err := r.createDesiredResource(isvc) - if err != nil { - return err - } + desiredResource := r.createDesiredResource(isvc) // Get Existing resource existingResource, err := r.getExistingResource(ctx, log, isvc) @@ -76,7 +75,7 @@ func (r *KservePrometheusRoleBindingReconciler) Cleanup(ctx context.Context, log return r.roleBindingHandler.DeleteRoleBinding(ctx, types.NamespacedName{Name: clusterPrometheusAccessRoleBinding, Namespace: isvcNs}) } -func (r *KservePrometheusRoleBindingReconciler) createDesiredResource(isvc *kservev1beta1.InferenceService) (*v1.RoleBinding, error) { +func (r *KservePrometheusRoleBindingReconciler) createDesiredResource(isvc *kservev1beta1.InferenceService) *v1.RoleBinding { desiredRoleBinding := &v1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: clusterPrometheusAccessRoleBinding, @@ -95,7 +94,7 @@ func (r *KservePrometheusRoleBindingReconciler) createDesiredResource(isvc *kser }, }, } - return desiredRoleBinding, nil + return desiredRoleBinding } func (r *KservePrometheusRoleBindingReconciler) getExistingResource(ctx context.Context, log logr.Logger, isvc *kservev1beta1.InferenceService) (*v1.RoleBinding, error) { @@ -114,7 +113,7 @@ func (r *KservePrometheusRoleBindingReconciler) processDelta(ctx context.Context if delta.IsAdded() { log.V(1).Info("Delta found", "create", desiredRoleBinding.GetName()) if err = r.client.Create(ctx, desiredRoleBinding); err != nil { - return + return err } } if delta.IsUpdated() { @@ -124,13 +123,13 @@ func (r *KservePrometheusRoleBindingReconciler) processDelta(ctx context.Context rp.Subjects = desiredRoleBinding.Subjects if err = r.client.Update(ctx, rp); err != nil { - return + return err } } if delta.IsRemoved() { log.V(1).Info("Delta found", "delete", existingRoleBinding.GetName()) if err = r.client.Delete(ctx, existingRoleBinding); err != nil { - return + return err } } return nil diff --git a/controllers/reconcilers/kserve_raw_clusterrolebinding_reconciler.go b/internal/controller/serving/reconcilers/kserve_raw_clusterrolebinding_reconciler.go similarity index 92% rename from controllers/reconcilers/kserve_raw_clusterrolebinding_reconciler.go rename to internal/controller/serving/reconcilers/kserve_raw_clusterrolebinding_reconciler.go index 38f45626..e3d61e1d 100644 --- a/controllers/reconcilers/kserve_raw_clusterrolebinding_reconciler.go +++ b/internal/controller/serving/reconcilers/kserve_raw_clusterrolebinding_reconciler.go @@ -20,12 +20,13 @@ import ( "github.com/go-logr/logr" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - "github.com/opendatahub-io/odh-model-controller/controllers/constants" - "github.com/opendatahub-io/odh-model-controller/controllers/processors" - "github.com/opendatahub-io/odh-model-controller/controllers/resources" v1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" + "github.com/opendatahub-io/odh-model-controller/internal/controller/processors" + "github.com/opendatahub-io/odh-model-controller/internal/controller/resources" ) var _ SubResourceReconciler = (*KserveRawClusterRoleBindingReconciler)(nil) @@ -49,10 +50,7 @@ func NewKserveRawClusterRoleBindingReconciler(client client.Client) *KserveRawCl func (r *KserveRawClusterRoleBindingReconciler) Reconcile(ctx context.Context, log logr.Logger, isvc *kservev1beta1.InferenceService) error { log.V(1).Info("Reconciling ClusterRoleBinding for InferenceService") // Create Desired resource - desiredResource, err := r.createDesiredResource(isvc) - if err != nil { - return err - } + desiredResource := r.createDesiredResource(isvc) // Get Existing resource existingResource, err := r.getExistingResource(ctx, log, isvc) @@ -102,14 +100,14 @@ func (r *KserveRawClusterRoleBindingReconciler) Delete(ctx context.Context, log return nil } -func (r *KserveRawClusterRoleBindingReconciler) createDesiredResource(isvc *kservev1beta1.InferenceService) (*v1.ClusterRoleBinding, error) { +func (r *KserveRawClusterRoleBindingReconciler) createDesiredResource(isvc *kservev1beta1.InferenceService) *v1.ClusterRoleBinding { isvcSA := r.serviceAccountName if isvc.Spec.Predictor.ServiceAccountName != "" { isvcSA = isvc.Spec.Predictor.ServiceAccountName } desiredClusterRoleBindingName := r.clusterRoleBindingHandler.GetClusterRoleBindingName(isvc.Namespace, isvcSA) desiredClusterRoleBinding := r.clusterRoleBindingHandler.CreateDesiredClusterRoleBinding(desiredClusterRoleBindingName, isvcSA, isvc.Namespace) - return desiredClusterRoleBinding, nil + return desiredClusterRoleBinding } func (r *KserveRawClusterRoleBindingReconciler) getExistingResource(ctx context.Context, log logr.Logger, isvc *kservev1beta1.InferenceService) (*v1.ClusterRoleBinding, error) { diff --git a/controllers/reconcilers/kserve_raw_inferenceservice_reconciler.go b/internal/controller/serving/reconcilers/kserve_raw_inferenceservice_reconciler.go similarity index 95% rename from controllers/reconcilers/kserve_raw_inferenceservice_reconciler.go rename to internal/controller/serving/reconcilers/kserve_raw_inferenceservice_reconciler.go index 2ac70587..f7bf4161 100644 --- a/controllers/reconcilers/kserve_raw_inferenceservice_reconciler.go +++ b/internal/controller/serving/reconcilers/kserve_raw_inferenceservice_reconciler.go @@ -18,13 +18,13 @@ package reconcilers import ( "context" - "github.com/hashicorp/go-multierror" - "github.com/opendatahub-io/odh-model-controller/controllers/constants" - "github.com/opendatahub-io/odh-model-controller/controllers/utils" - "github.com/go-logr/logr" + "github.com/hashicorp/go-multierror" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" ) var _ Reconciler = (*KserveRawInferenceServiceReconciler)(nil) diff --git a/controllers/reconcilers/kserve_raw_route_reconciler.go b/internal/controller/serving/reconcilers/kserve_raw_route_reconciler.go similarity index 93% rename from controllers/reconcilers/kserve_raw_route_reconciler.go rename to internal/controller/serving/reconcilers/kserve_raw_route_reconciler.go index b7284c60..b9a9aa4b 100644 --- a/controllers/reconcilers/kserve_raw_route_reconciler.go +++ b/internal/controller/serving/reconcilers/kserve_raw_route_reconciler.go @@ -21,18 +21,19 @@ import ( "github.com/go-logr/logr" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - "github.com/opendatahub-io/odh-model-controller/controllers/comparators" - "github.com/opendatahub-io/odh-model-controller/controllers/constants" - "github.com/opendatahub-io/odh-model-controller/controllers/processors" - "github.com/opendatahub-io/odh-model-controller/controllers/resources" v1 "github.com/openshift/api/route/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/comparators" + "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" + "github.com/opendatahub-io/odh-model-controller/internal/controller/processors" + "github.com/opendatahub-io/odh-model-controller/internal/controller/resources" ) var _ SubResourceReconciler = (*KserveRawRouteReconciler)(nil) @@ -145,7 +146,7 @@ func (r *KserveRawRouteReconciler) createDesiredResource(ctx context.Context, lo To: v1.RouteTargetReference{ Kind: "Service", Name: targetService.Name, - Weight: pointer.Int32(100), + Weight: ptr.To(int32(100)), }, Port: &v1.RoutePort{ TargetPort: intstr.FromInt32(servicePort), @@ -191,7 +192,7 @@ func (r *KserveRawRouteReconciler) processDelta(ctx context.Context, log logr.Lo if delta.IsAdded() { log.V(1).Info("Delta found", "create", desiredRoute.GetName()) if err = r.client.Create(ctx, desiredRoute); err != nil { - return + return err } } if delta.IsUpdated() { @@ -202,13 +203,13 @@ func (r *KserveRawRouteReconciler) processDelta(ctx context.Context, log logr.Lo rp.Spec = desiredRoute.Spec if err = r.client.Update(ctx, rp); err != nil { - return + return err } } if delta.IsRemoved() { log.V(1).Info("Delta found", "delete", existingRoute.GetName()) if err = r.client.Delete(ctx, existingRoute); err != nil { - return + return err } } return nil diff --git a/controllers/reconcilers/kserve_route_reconciler.go b/internal/controller/serving/reconcilers/kserve_route_reconciler.go similarity index 86% rename from controllers/reconcilers/kserve_route_reconciler.go rename to internal/controller/serving/reconcilers/kserve_route_reconciler.go index 97491240..50ef13c2 100644 --- a/controllers/reconcilers/kserve_route_reconciler.go +++ b/internal/controller/serving/reconcilers/kserve_route_reconciler.go @@ -18,35 +18,40 @@ package reconcilers import ( "context" "fmt" + "github.com/go-logr/logr" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" "github.com/kserve/kserve/pkg/constants" "github.com/kserve/kserve/pkg/utils" - "github.com/opendatahub-io/odh-model-controller/controllers/comparators" - constants2 "github.com/opendatahub-io/odh-model-controller/controllers/constants" - "github.com/opendatahub-io/odh-model-controller/controllers/processors" - "github.com/opendatahub-io/odh-model-controller/controllers/resources" - utils2 "github.com/opendatahub-io/odh-model-controller/controllers/utils" v1 "github.com/openshift/api/route/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/utils/pointer" + "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" "knative.dev/pkg/network" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/comparators" + constants2 "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" + "github.com/opendatahub-io/odh-model-controller/internal/controller/processors" + "github.com/opendatahub-io/odh-model-controller/internal/controller/resources" + utils2 "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" ) var _ SubResourceReconciler = (*KserveRouteReconciler)(nil) type KserveRouteReconciler struct { client client.Client + kClient kubernetes.Interface routeHandler resources.RouteHandler deltaProcessor processors.DeltaProcessor } -func NewKserveRouteReconciler(client client.Client) *KserveRouteReconciler { +func NewKserveRouteReconciler(client client.Client, kClient kubernetes.Interface) *KserveRouteReconciler { return &KserveRouteReconciler{ client: client, + kClient: kClient, routeHandler: resources.NewRouteHandler(client), deltaProcessor: processors.NewDeltaProcessor(), } @@ -86,20 +91,20 @@ func (r *KserveRouteReconciler) Cleanup(_ context.Context, _ logr.Logger, _ stri } func (r *KserveRouteReconciler) createDesiredResource(isvc *kservev1beta1.InferenceService) (*v1.Route, error) { - ingressConfig, err := kservev1beta1.NewIngressConfig(r.client) + ingressConfig, err := kservev1beta1.NewIngressConfig(r.kClient) if err != nil { return nil, err } disableIstioVirtualHost := ingressConfig.DisableIstioVirtualHost - if disableIstioVirtualHost == false { + if !disableIstioVirtualHost { serviceHost := getServiceHost(isvc) if serviceHost == "" { return nil, fmt.Errorf("failed to load serviceHost from InferenceService status") } isInternal := false - //if service is labelled with cluster local or knative domain is configured as internal + // if service is labelled with cluster local or knative domain is configured as internal if val, ok := isvc.Labels[constants.VisibilityLabel]; ok && val == constants.ClusterLocalVisibility { isInternal = true } @@ -145,7 +150,7 @@ func (r *KserveRouteReconciler) createDesiredResource(isvc *kservev1beta1.Infere To: v1.RouteTargetReference{ Kind: "Service", Name: constants2.IstioIngressService, - Weight: pointer.Int32(100), + Weight: ptr.To(int32(100)), }, Port: &v1.RoutePort{ TargetPort: targetPort, @@ -153,9 +158,6 @@ func (r *KserveRouteReconciler) createDesiredResource(isvc *kservev1beta1.Infere TLS: tlsConfig, WildcardPolicy: v1.WildcardPolicyNone, }, - Status: v1.RouteStatus{ - Ingress: []v1.RouteIngress{}, - }, } return route, nil } @@ -179,7 +181,7 @@ func (r *KserveRouteReconciler) processDelta(ctx context.Context, log logr.Logge if delta.IsAdded() { log.V(1).Info("Delta found", "create", desiredRoute.GetName()) if err = r.client.Create(ctx, desiredRoute); err != nil { - return + return err } } if delta.IsUpdated() { @@ -190,13 +192,13 @@ func (r *KserveRouteReconciler) processDelta(ctx context.Context, log logr.Logge rp.Spec = desiredRoute.Spec if err = r.client.Update(ctx, rp); err != nil { - return + return err } } if delta.IsRemoved() { log.V(1).Info("Delta found", "delete", existingRoute.GetName()) if err = r.client.Delete(ctx, existingRoute); err != nil { - return + return err } } return nil @@ -206,7 +208,7 @@ func getServiceHost(isvc *kservev1beta1.InferenceService) string { if isvc.Status.URL == nil { return "" } - //Derive the ingress service host from underlying service url + // Derive the ingress service host from underlying service url return isvc.Status.URL.Host } diff --git a/controllers/reconcilers/kserve_serverless_inferenceservice_reconciler.go b/internal/controller/serving/reconcilers/kserve_serverless_inferenceservice_reconciler.go similarity index 91% rename from controllers/reconcilers/kserve_serverless_inferenceservice_reconciler.go rename to internal/controller/serving/reconcilers/kserve_serverless_inferenceservice_reconciler.go index 956592cf..34585512 100644 --- a/controllers/reconcilers/kserve_serverless_inferenceservice_reconciler.go +++ b/internal/controller/serving/reconcilers/kserve_serverless_inferenceservice_reconciler.go @@ -21,9 +21,11 @@ import ( "github.com/go-logr/logr" "github.com/hashicorp/go-multierror" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - constants "github.com/opendatahub-io/odh-model-controller/controllers/constants" - "github.com/opendatahub-io/odh-model-controller/controllers/utils" + "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" + + constants "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" ) var _ Reconciler = (*KserveServerlessInferenceServiceReconciler)(nil) @@ -33,11 +35,11 @@ type KserveServerlessInferenceServiceReconciler struct { subResourceReconcilers []SubResourceReconciler } -func NewKServeServerlessInferenceServiceReconciler(client client.Client, clientReader client.Reader) *KserveServerlessInferenceServiceReconciler { +func NewKServeServerlessInferenceServiceReconciler(client client.Client, clientReader client.Reader, kClient kubernetes.Interface) *KserveServerlessInferenceServiceReconciler { subResourceReconciler := []SubResourceReconciler{ NewKserveServiceMeshMemberReconciler(client), - NewKserveRouteReconciler(client), + NewKserveRouteReconciler(client, kClient), NewKServeMetricsServiceReconciler(client), NewKServeMetricsServiceMonitorReconciler(client), NewKServePrometheusRoleBindingReconciler(client), diff --git a/controllers/reconcilers/mm_clusterrolebinding_reconciler.go b/internal/controller/serving/reconcilers/mm_clusterrolebinding_reconciler.go similarity index 89% rename from controllers/reconcilers/mm_clusterrolebinding_reconciler.go rename to internal/controller/serving/reconcilers/mm_clusterrolebinding_reconciler.go index 922f981d..7562f63d 100644 --- a/controllers/reconcilers/mm_clusterrolebinding_reconciler.go +++ b/internal/controller/serving/reconcilers/mm_clusterrolebinding_reconciler.go @@ -20,12 +20,13 @@ import ( "github.com/go-logr/logr" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - "github.com/opendatahub-io/odh-model-controller/controllers/constants" - "github.com/opendatahub-io/odh-model-controller/controllers/processors" - "github.com/opendatahub-io/odh-model-controller/controllers/resources" v1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" + "github.com/opendatahub-io/odh-model-controller/internal/controller/processors" + "github.com/opendatahub-io/odh-model-controller/internal/controller/resources" ) var _ SubResourceReconciler = (*ModelMeshClusterRoleBindingReconciler)(nil) @@ -50,10 +51,7 @@ func NewModelMeshClusterRoleBindingReconciler(client client.Client) *ModelMeshCl func (r *ModelMeshClusterRoleBindingReconciler) Reconcile(ctx context.Context, log logr.Logger, isvc *kservev1beta1.InferenceService) error { log.V(1).Info("Reconciling ClusterRoleBinding for InferenceService") // Create Desired resource - desiredResource, err := r.createDesiredResource(isvc) - if err != nil { - return err - } + desiredResource := r.createDesiredResource(isvc) // Get Existing resource existingResource, err := r.getExistingResource(ctx, log, isvc) @@ -74,10 +72,10 @@ func (r *ModelMeshClusterRoleBindingReconciler) Cleanup(ctx context.Context, log return r.clusterRoleBindingHandler.DeleteClusterRoleBinding(ctx, types.NamespacedName{Name: crbName, Namespace: isvcNs}) } -func (r *ModelMeshClusterRoleBindingReconciler) createDesiredResource(isvc *kservev1beta1.InferenceService) (*v1.ClusterRoleBinding, error) { +func (r *ModelMeshClusterRoleBindingReconciler) createDesiredResource(isvc *kservev1beta1.InferenceService) *v1.ClusterRoleBinding { desiredClusterRoleBindingName := r.clusterRoleBindingHandler.GetClusterRoleBindingName(isvc.Namespace, r.serviceAccountName) desiredClusterRoleBinding := r.clusterRoleBindingHandler.CreateDesiredClusterRoleBinding(desiredClusterRoleBindingName, r.serviceAccountName, isvc.Namespace) - return desiredClusterRoleBinding, nil + return desiredClusterRoleBinding } func (r *ModelMeshClusterRoleBindingReconciler) getExistingResource(ctx context.Context, log logr.Logger, isvc *kservev1beta1.InferenceService) (*v1.ClusterRoleBinding, error) { diff --git a/controllers/reconcilers/mm_inferenceservice_reconciler.go b/internal/controller/serving/reconcilers/mm_inferenceservice_reconciler.go similarity index 94% rename from controllers/reconcilers/mm_inferenceservice_reconciler.go rename to internal/controller/serving/reconcilers/mm_inferenceservice_reconciler.go index 521d6b5c..0d6a4ad3 100644 --- a/controllers/reconcilers/mm_inferenceservice_reconciler.go +++ b/internal/controller/serving/reconcilers/mm_inferenceservice_reconciler.go @@ -18,13 +18,13 @@ package reconcilers import ( "context" - "github.com/hashicorp/go-multierror" - "github.com/opendatahub-io/odh-model-controller/controllers/constants" - "github.com/go-logr/logr" + "github.com/hashicorp/go-multierror" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - "github.com/opendatahub-io/odh-model-controller/controllers/utils" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" ) var _ Reconciler = (*ModelMeshInferenceServiceReconciler)(nil) diff --git a/controllers/reconcilers/mm_route_reconciler.go b/internal/controller/serving/reconcilers/mm_route_reconciler.go similarity index 92% rename from controllers/reconcilers/mm_route_reconciler.go rename to internal/controller/serving/reconcilers/mm_route_reconciler.go index a5981190..46b2ec9d 100644 --- a/controllers/reconcilers/mm_route_reconciler.go +++ b/internal/controller/serving/reconcilers/mm_route_reconciler.go @@ -23,18 +23,19 @@ import ( "github.com/go-logr/logr" kservev1alpha1 "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - "github.com/opendatahub-io/odh-model-controller/controllers/comparators" - "github.com/opendatahub-io/odh-model-controller/controllers/constants" - "github.com/opendatahub-io/odh-model-controller/controllers/processors" - "github.com/opendatahub-io/odh-model-controller/controllers/resources" v1 "github.com/openshift/api/route/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/comparators" + "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" + "github.com/opendatahub-io/odh-model-controller/internal/controller/processors" + "github.com/opendatahub-io/odh-model-controller/internal/controller/resources" ) const ( @@ -115,7 +116,7 @@ func (r *ModelMeshRouteReconciler) createDesiredResource(ctx context.Context, lo To: v1.RouteTargetReference{ Kind: "Service", Name: modelmeshServiceName, - Weight: pointer.Int32(100), + Weight: ptr.To(int32(100)), }, Port: &v1.RoutePort{ TargetPort: intstr.FromInt(modelmeshServicePort), @@ -164,7 +165,7 @@ func (r *ModelMeshRouteReconciler) processDelta(ctx context.Context, log logr.Lo if delta.IsAdded() { log.V(1).Info("Delta found", "create", desiredRoute.GetName()) if err = r.client.Create(ctx, desiredRoute); err != nil { - return + return err } } if delta.IsUpdated() { @@ -175,13 +176,13 @@ func (r *ModelMeshRouteReconciler) processDelta(ctx context.Context, log logr.Lo rp.Spec = desiredRoute.Spec if err = r.client.Update(ctx, rp); err != nil { - return + return err } } if delta.IsRemoved() { log.V(1).Info("Delta found", "delete", existingRoute.GetName()) if err = r.client.Delete(ctx, existingRoute); err != nil { - return + return err } } return nil @@ -222,16 +223,16 @@ func (r *ModelMeshRouteReconciler) findSupportingRuntimeForISvc(ctx context.Cont }) for _, runtime := range runtimes.Items { - if runtime.Spec.Disabled != nil && *runtime.Spec.Disabled == true { + if runtime.Spec.Disabled != nil && *runtime.Spec.Disabled { continue } - if runtime.Spec.MultiModel != nil && *runtime.Spec.MultiModel == false { + if runtime.Spec.MultiModel != nil && !*runtime.Spec.MultiModel { continue } for _, supportedFormat := range runtime.Spec.SupportedModelFormats { - if supportedFormat.AutoSelect != nil && *supportedFormat.AutoSelect == true && supportedFormat.Name == isvc.Spec.Predictor.Model.ModelFormat.Name { + if supportedFormat.AutoSelect != nil && *supportedFormat.AutoSelect && supportedFormat.Name == isvc.Spec.Predictor.Model.ModelFormat.Name { desiredServingRuntime = &runtime log.Info("Automatic runtime selection for InferenceService", "runtime", desiredServingRuntime.Name) return desiredServingRuntime, nil diff --git a/controllers/mr_inferenceservice_controller.go b/internal/controller/serving/reconcilers/model_registry_reconciler.go similarity index 88% rename from controllers/mr_inferenceservice_controller.go rename to internal/controller/serving/reconcilers/model_registry_reconciler.go index 77255945..1c253810 100644 --- a/controllers/mr_inferenceservice_controller.go +++ b/internal/controller/serving/reconcilers/model_registry_reconciler.go @@ -1,14 +1,15 @@ -package controllers +package reconcilers import ( "context" "github.com/go-logr/logr" infrctrl "github.com/kubeflow/model-registry/pkg/inferenceservice-controller" - "github.com/opendatahub-io/odh-model-controller/controllers/constants" - "github.com/opendatahub-io/odh-model-controller/controllers/utils" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" ) func NewModelRegistryInferenceServiceReconciler(client client.Client, log logr.Logger, skipTLSVerify bool, bearerToken string) *infrctrl.InferenceServiceController { diff --git a/controllers/reconcilers/serviceaccount_reconciler.go b/internal/controller/serving/reconcilers/serviceaccount_reconciler.go similarity index 90% rename from controllers/reconcilers/serviceaccount_reconciler.go rename to internal/controller/serving/reconcilers/serviceaccount_reconciler.go index 11c00925..39575e1d 100644 --- a/controllers/reconcilers/serviceaccount_reconciler.go +++ b/internal/controller/serving/reconcilers/serviceaccount_reconciler.go @@ -19,16 +19,15 @@ import ( "context" "github.com/go-logr/logr" - "github.com/opendatahub-io/odh-model-controller/controllers/comparators" - "github.com/opendatahub-io/odh-model-controller/controllers/processors" - "github.com/opendatahub-io/odh-model-controller/controllers/resources" - "sigs.k8s.io/controller-runtime/pkg/client" - - "k8s.io/apimachinery/pkg/types" - kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/comparators" + "github.com/opendatahub-io/odh-model-controller/internal/controller/processors" + "github.com/opendatahub-io/odh-model-controller/internal/controller/resources" ) var _ SubResourceReconciler = (*ServiceAccountReconciler)(nil) @@ -53,10 +52,7 @@ func NewServiceAccountReconciler(client client.Client, serviceAccountName string func (r *ServiceAccountReconciler) Reconcile(ctx context.Context, log logr.Logger, isvc *kservev1beta1.InferenceService) error { log.V(1).Info("Reconciling ServiceAccount for InferenceService") // Create Desired resource - desiredResource, err := r.createDesiredResource(isvc) - if err != nil { - return err - } + desiredResource := r.createDesiredResource(isvc) // Get Existing resource existingResource, err := r.getExistingResource(ctx, log, isvc) @@ -76,14 +72,14 @@ func (r *ServiceAccountReconciler) Cleanup(ctx context.Context, log logr.Logger, return r.serviceAccountHandler.DeleteServiceAccount(ctx, types.NamespacedName{Name: r.serviceAccountName, Namespace: isvcNs}) } -func (r *ServiceAccountReconciler) createDesiredResource(isvc *kservev1beta1.InferenceService) (*corev1.ServiceAccount, error) { +func (r *ServiceAccountReconciler) createDesiredResource(isvc *kservev1beta1.InferenceService) *corev1.ServiceAccount { desiredSA := &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Name: r.serviceAccountName, Namespace: isvc.Namespace, }, } - return desiredSA, nil + return desiredSA } func (r *ServiceAccountReconciler) getExistingResource(ctx context.Context, log logr.Logger, isvc *kservev1beta1.InferenceService) (*corev1.ServiceAccount, error) { diff --git a/controllers/reconcilers/types.go b/internal/controller/serving/reconcilers/types.go similarity index 99% rename from controllers/reconcilers/types.go rename to internal/controller/serving/reconcilers/types.go index cc0476b5..83d98c49 100644 --- a/controllers/reconcilers/types.go +++ b/internal/controller/serving/reconcilers/types.go @@ -17,6 +17,7 @@ package reconcilers import ( "context" + "github.com/go-logr/logr" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" ) diff --git a/controllers/monitoring_controller.go b/internal/controller/serving/servingruntime_controller.go similarity index 68% rename from controllers/monitoring_controller.go rename to internal/controller/serving/servingruntime_controller.go index 2bcbd791..c67ff106 100644 --- a/controllers/monitoring_controller.go +++ b/internal/controller/serving/servingruntime_controller.go @@ -1,4 +1,5 @@ /* +Copyright 2024. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -13,21 +14,24 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controllers +package serving import ( "context" - "github.com/go-logr/logr" + "reflect" + kservev1alpha1 "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" + servingv1alpha1 "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" corev1 "k8s.io/api/core/v1" k8srbacv1 "k8s.io/api/rbac/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "reflect" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -39,12 +43,17 @@ const ( MonitoringSA = "prometheus-custom" ) -type MonitoringReconciler struct { +// ServingRuntimeReconciler reconciles a ServingRuntime object. Formerly +// known as MonitoringReconciler. +type ServingRuntimeReconciler struct { client.Client - Log logr.Logger + Scheme *runtime.Scheme MonitoringNS string } +// +kubebuilder:rbac:groups=serving.kserve.io,resources=servingruntimes,verbs=get;list;watch;create;update +// +kubebuilder:rbac:groups=serving.kserve.io,resources=servingruntimes/finalizers,verbs=update + // RoleBindingsAreEqual checks if RoleBinding are equal, if not return false func RoleBindingsAreEqual(sm1 k8srbacv1.RoleBinding, sm2 k8srbacv1.RoleBinding) bool { areEqual := @@ -75,7 +84,8 @@ func buildDesiredRB(rbNS string, monitoringNS string) *k8srbacv1.RoleBinding { } // foundRB stores monitoring rbac in actualRB if it is found in ns namespace -func (r *MonitoringReconciler) foundRB(ctx context.Context, actualRB *k8srbacv1.RoleBinding, ns string) (bool, error) { +func (r *ServingRuntimeReconciler) foundRB(ctx context.Context, actualRB *k8srbacv1.RoleBinding, ns string) (bool, error) { + logger := log.FromContext(ctx) namespacedName := types.NamespacedName{ Name: RoleBindingName, Namespace: ns, @@ -84,21 +94,22 @@ func (r *MonitoringReconciler) foundRB(ctx context.Context, actualRB *k8srbacv1. if apierrs.IsNotFound(err) { return false, nil } else if err != nil { - r.Log.Error(err, "Failed to get rolebinding"+RoleBindingName) + logger.Error(err, "Failed to get rolebinding"+RoleBindingName) return false, err } return true, nil } // createRBIfDNE will attempt to create desiredRB if it does not exist, or is different from actualRB -func (r *MonitoringReconciler) createRBIfDNE(ctx context.Context, exists bool, desiredRB, actualRB *k8srbacv1.RoleBinding) error { +func (r *ServingRuntimeReconciler) createRBIfDNE(ctx context.Context, exists bool, desiredRB, actualRB *k8srbacv1.RoleBinding) error { + logger := log.FromContext(ctx) if !exists { err := r.Create(ctx, desiredRB) if err != nil { - r.Log.Error(err, "Failed to create Rolebinding"+RoleBindingName) + logger.Error(err, "Failed to create Rolebinding"+RoleBindingName) return err } - r.Log.Info("Created RoleBinding: " + RoleBindingName) + logger.Info("Created RoleBinding: " + RoleBindingName) return nil } @@ -112,18 +123,18 @@ func (r *MonitoringReconciler) createRBIfDNE(ctx context.Context, exists bool, d err := r.Client.Update(ctx, desiredRB) if apierrs.IsConflict(err) { // may occur during if the RoleBinding was updated during this reconcile loop - r.Log.Error(err, "Failed to create/update RoleBinding: "+RoleBindingName+" due to resource conflict") + logger.Error(err, "Failed to create/update RoleBinding: "+RoleBindingName+" due to resource conflict") return err } else if err != nil { - r.Log.Error(err, "Failed to create/update RoleBinding: "+RoleBindingName) + logger.Error(err, "Failed to create/update RoleBinding: "+RoleBindingName) return err } - r.Log.Info("Updated RoleBinding: " + RoleBindingName) + logger.Info("Updated RoleBinding: " + RoleBindingName) return nil } // modelMeshEnabled return true if this Namespace is modelmesh enabled -func (r *MonitoringReconciler) modelMeshEnabled(ns string, labels map[string]string) bool { +func (r *ServingRuntimeReconciler) modelMeshEnabled(_ string, labels map[string]string) bool { enabled, ok := labels["modelmesh-enabled"] if !ok || enabled != "true" { return false @@ -132,15 +143,15 @@ func (r *MonitoringReconciler) modelMeshEnabled(ns string, labels map[string]str } // monitoringThisNameSpace return true if this Namespace should be monitored by monitoring stack -func (r *MonitoringReconciler) monitoringThisNameSpace(ns string, labels map[string]string) bool { +func (r *ServingRuntimeReconciler) monitoringThisNameSpace(ns string, labels map[string]string) bool { if ns == OpenshiftMonitoringNS || ns == r.MonitoringNS { return true } return r.modelMeshEnabled(ns, labels) } -func (r *MonitoringReconciler) reconcileRoleBinding(ctx context.Context, req ctrl.Request) error { - log := r.Log.WithValues("ResourceName", req.Name, "Namespace", req.Namespace) +func (r *ServingRuntimeReconciler) reconcileRoleBinding(ctx context.Context, req ctrl.Request) error { + logger := log.FromContext(ctx) ns := &corev1.Namespace{} err := r.Client.Get(ctx, types.NamespacedName{Name: req.Namespace}, ns) @@ -151,7 +162,7 @@ func (r *MonitoringReconciler) reconcileRoleBinding(ctx context.Context, req ctr monitoringNS := r.monitoringThisNameSpace(req.Namespace, ns.Labels) if !monitoringNS { - log.Info("Namespace is not modelmesh enabled, or configured for monitoring, skipping.") + logger.Info("Namespace is not modelmesh enabled, or configured for monitoring, skipping.") return nil } @@ -183,7 +194,7 @@ func (r *MonitoringReconciler) reconcileRoleBinding(ctx context.Context, req ctr if apierrs.IsNotFound(err) { noServingRuntimes = true } else { - log.Error(err, "Unable to fetch the ServingRuntimes") + logger.Error(err, "Unable to fetch the ServingRuntimes") return err } } @@ -200,10 +211,10 @@ func (r *MonitoringReconciler) reconcileRoleBinding(ctx context.Context, req ctr if roleBindingExists { err := r.Delete(ctx, actualRB) if err != nil { - log.Error(err, "Failed to delete monitoring Rolebinding"+RoleBindingName) + logger.Error(err, "Failed to delete monitoring Rolebinding"+RoleBindingName) return err } - log.Info("No Serving Runtimes detected in this namespace, deleted monitoring RoleBinding : " + RoleBindingName) + logger.Info("No Serving Runtimes detected in this namespace, deleted monitoring RoleBinding : " + RoleBindingName) } return nil } @@ -220,12 +231,22 @@ func (r *MonitoringReconciler) reconcileRoleBinding(ctx context.Context, req ctr return nil } -// Reconcile will manage the creation, update and deletion of the ModelMesh monitoring resources -func (r *MonitoringReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the ServingRuntime object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/reconcile +func (r *ServingRuntimeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + _ = log.FromContext(ctx) // Initialize logger format - log := r.Log.WithValues("ResourceName", req.Name, "Namespace", req.Namespace) + logger := log.FromContext(ctx).WithValues("ResourceName", req.Name, "Namespace", req.Namespace) + ctx = log.IntoContext(ctx, logger) if r.MonitoringNS == "" { - log.Info("No monitoring namespace detected, skipping monitoring reconciliation.") + logger.Info("No monitoring namespace detected, skipping monitoring reconciliation.") return ctrl.Result{}, nil } @@ -238,19 +259,20 @@ func (r *MonitoringReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } - log.Info("Monitoring Controller reconciling.") + logger.Info("Monitoring Controller reconciling.") err = r.reconcileRoleBinding(ctx, req) if err != nil { return ctrl.Result{}, err } - log.Info("Monitoring Controller reconciled successfully.") + logger.Info("Monitoring Controller reconciled successfully.") return ctrl.Result{}, nil } // SetupWithManager sets up the controller with the Manager. -func (r *MonitoringReconciler) SetupWithManager(mgr ctrl.Manager) error { - builder := ctrl.NewControllerManagedBy(mgr). - For(&kservev1alpha1.ServingRuntime{}). +func (r *ServingRuntimeReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&servingv1alpha1.ServingRuntime{}). + Named("servingruntime"). // Watch for changes to ModelMesh Enabled namespaces & a select few others Watches(&corev1.Namespace{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []reconcile.Request { @@ -268,6 +290,7 @@ func (r *MonitoringReconciler) SetupWithManager(mgr ctrl.Manager) error { // Watch for RoleBinding in modelmesh enabled namespaces & a select few others Watches(&k8srbacv1.RoleBinding{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []reconcile.Request { + logger := log.FromContext(ctx) // Only reconcile on RoleBindings that this controller creates. // We avoid using owner references, as there is no logical owner // of the RoleBinding this controller creates. @@ -278,7 +301,7 @@ func (r *MonitoringReconciler) SetupWithManager(mgr ctrl.Manager) error { if !odhManaged { return []reconcile.Request{} } - r.Log.Info("Reconcile event triggered by Rolebinding: " + o.GetName()) + logger.Info("Reconcile event triggered by Rolebinding: " + o.GetName()) namespacedName := types.NamespacedName{ Name: o.GetName(), @@ -288,10 +311,6 @@ func (r *MonitoringReconciler) SetupWithManager(mgr ctrl.Manager) error { reconcileRequests := append([]reconcile.Request{}, reconcile.Request{NamespacedName: namespacedName}) return reconcileRequests - })) - err := builder.Complete(r) - if err != nil { - return err - } - return nil + })). + Complete(r) } diff --git a/controllers/monitoring_controller_test.go b/internal/controller/serving/servingruntime_controller_test.go similarity index 75% rename from controllers/monitoring_controller_test.go rename to internal/controller/serving/servingruntime_controller_test.go index b812c51f..162d867e 100644 --- a/controllers/monitoring_controller_test.go +++ b/internal/controller/serving/servingruntime_controller_test.go @@ -1,4 +1,5 @@ /* +Copyright 2024. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -13,45 +14,45 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controllers +package serving import ( "context" "errors" + kservev1alpha1 "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" k8srbacv1 "k8s.io/api/rbac/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + + testutils "github.com/opendatahub-io/odh-model-controller/test/utils" ) func deployServingRuntime(path string, ctx context.Context) { servingRuntime := &kservev1alpha1.ServingRuntime{} - err := convertToStructuredResource(path, servingRuntime) + err := testutils.ConvertToStructuredResource(path, servingRuntime) Expect(err).NotTo(HaveOccurred()) - Expect(cli.Create(ctx, servingRuntime)).Should(Succeed()) + Expect(k8sClient.Create(ctx, servingRuntime)).Should(Succeed()) } func deleteServingRuntime(path string, ctx context.Context) { servingRuntime := &kservev1alpha1.ServingRuntime{} - err := convertToStructuredResource(path, servingRuntime) + err := testutils.ConvertToStructuredResource(path, servingRuntime) Expect(err).NotTo(HaveOccurred()) - Expect(cli.Delete(ctx, servingRuntime)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, servingRuntime)).Should(Succeed()) } -var _ = Describe("ODH Controller's Monitoring Controller", func() { - - ctx := context.Background() - +var _ = Describe("ServingRuntime Controller (ODH Monitoring Controller)", func() { Context("In a modelmesh enabled namespace", func() { BeforeEach(func() { ns := &corev1.Namespace{} - Expect(cli.Get(ctx, types.NamespacedName{Name: WorkingNamespace}, ns)).NotTo(HaveOccurred()) + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: WorkingNamespace}, ns)).NotTo(HaveOccurred()) ns.Labels["modelmesh-enabled"] = "true" Eventually(func() error { - return cli.Update(ctx, ns) + return k8sClient.Update(ctx, ns) }, timeout, interval).ShouldNot(HaveOccurred()) }) @@ -62,23 +63,23 @@ var _ = Describe("ODH Controller's Monitoring Controller", func() { deployServingRuntime(ServingRuntimePath1, ctx) expectedRB := &k8srbacv1.RoleBinding{} - Expect(convertToStructuredResource(RoleBindingPath, expectedRB)).NotTo(HaveOccurred()) + Expect(testutils.ConvertToStructuredResource(RoleBindingPath, expectedRB)).NotTo(HaveOccurred()) expectedRB.Subjects[0].Namespace = MonitoringNS actualRB := &k8srbacv1.RoleBinding{} Eventually(func() error { namespacedNamed := types.NamespacedName{Name: expectedRB.Name, Namespace: WorkingNamespace} - return cli.Get(ctx, namespacedNamed, actualRB) + return k8sClient.Get(ctx, namespacedNamed, actualRB) }, timeout, interval).ShouldNot(HaveOccurred()) Expect(RoleBindingsAreEqual(*expectedRB, *actualRB)).Should(BeTrue()) By("create the Monitoring Rolebinding if it is removed.") - Expect(cli.Delete(ctx, actualRB)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, actualRB)).Should(Succeed()) Eventually(func() error { namespacedNamed := types.NamespacedName{Name: expectedRB.Name, Namespace: WorkingNamespace} - return cli.Get(ctx, namespacedNamed, actualRB) + return k8sClient.Get(ctx, namespacedNamed, actualRB) }, timeout, interval).ShouldNot(HaveOccurred()) Expect(RoleBindingsAreEqual(*expectedRB, *actualRB)).Should(BeTrue()) @@ -88,7 +89,7 @@ var _ = Describe("ODH Controller's Monitoring Controller", func() { deleteServingRuntime(ServingRuntimePath1, ctx) Eventually(func() error { namespacedNamed := types.NamespacedName{Name: expectedRB.Name, Namespace: WorkingNamespace} - return cli.Get(ctx, namespacedNamed, actualRB) + return k8sClient.Get(ctx, namespacedNamed, actualRB) }, timeout, interval).ShouldNot(HaveOccurred()) Expect(RoleBindingsAreEqual(*expectedRB, *actualRB)).Should(BeTrue()) @@ -97,7 +98,7 @@ var _ = Describe("ODH Controller's Monitoring Controller", func() { deleteServingRuntime(ServingRuntimePath2, ctx) Eventually(func() error { namespacedNamed := types.NamespacedName{Name: expectedRB.Name, Namespace: WorkingNamespace} - err := cli.Get(ctx, namespacedNamed, actualRB) + err := k8sClient.Get(ctx, namespacedNamed, actualRB) if apierrs.IsNotFound(err) { return nil } else { diff --git a/internal/controller/serving/suite_test.go b/internal/controller/serving/suite_test.go new file mode 100644 index 00000000..01124832 --- /dev/null +++ b/internal/controller/serving/suite_test.go @@ -0,0 +1,217 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package serving + +import ( + "context" + "fmt" + "path/filepath" + "runtime" + "testing" + "time" + + kservev1alpha1 "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" + kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" + authorinov1beta2 "github.com/kuadrant/authorino/api/v1beta2" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + routev1 "github.com/openshift/api/route/v1" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + istioclientv1beta1 "istio.io/client-go/pkg/apis/networking/v1beta1" + istiov1beta1 "istio.io/client-go/pkg/apis/security/v1beta1" + corev1 "k8s.io/api/core/v1" + k8srbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sLabels "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + // +kubebuilder:scaffold:imports + + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" + testutils "github.com/opendatahub-io/odh-model-controller/test/utils" +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var cfg *rest.Config +var k8sClient client.Client +var testEnv *envtest.Environment +var ctx context.Context +var cancel context.CancelFunc + +const ( + WorkingNamespace = "default" + MonitoringNS = "monitoring-ns" + RoleBindingPath = "./testdata/results/model-server-ns-role.yaml" + ServingRuntimePath1 = "./testdata/deploy/test-openvino-serving-runtime-1.yaml" + KserveServingRuntimePath1 = "./testdata/deploy/kserve-openvino-serving-runtime-1.yaml" + ServingRuntimePath2 = "./testdata/deploy/test-openvino-serving-runtime-2.yaml" + InferenceService1 = "./testdata/deploy/openvino-inference-service-1.yaml" + InferenceServiceNoRuntime = "./testdata/deploy/openvino-inference-service-no-runtime.yaml" + KserveInferenceServicePath1 = "./testdata/deploy/kserve-openvino-inference-service-1.yaml" + InferenceServiceConfigPath1 = "./testdata/configmaps/inferenceservice-config.yaml" + ExpectedRoutePath = "./testdata/results/example-onnx-mnist-route.yaml" + ExpectedRouteNoRuntimePath = "./testdata/results/example-onnx-mnist-no-runtime-route.yaml" + DSCIWithAuthorization = "./testdata/dsci-with-authorino-enabled.yaml" + DSCIWithoutAuthorization = "./testdata/dsci-with-authorino-missing.yaml" + KServeAuthorizationPolicy = "./testdata/kserve-authorization-policy.yaml" + odhtrustedcabundleConfigMapPath = "./testdata/configmaps/odh-trusted-ca-bundle-configmap.yaml" + timeout = time.Second * 20 + interval = time.Millisecond * 10 +) + +func TestControllers(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Controller Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{ + filepath.Join("..", "..", "..", "config", "crd", "bases"), + filepath.Join("..", "..", "..", "config", "crd", "external"), + }, + ErrorIfCRDPathMissing: true, + + // The BinaryAssetsDirectory is only required if you want to run the tests directly + // without call the makefile target test. If not informed it will look for the + // default path defined in controller-runtime which is /usr/local/kubebuilder/. + // Note that you must have the required binaries setup under the bin directory to perform + // the tests directly. When we run make test it will be setup and used automatically. + BinaryAssetsDirectory: filepath.Join("..", "..", "..", "bin", "k8s", + fmt.Sprintf("1.31.0-%s-%s", runtime.GOOS, runtime.GOARCH)), + } + + var err error + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + utils.RegisterSchemes(scheme.Scheme) + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + kubeClient, kubeClientErr := kubernetes.NewForConfig(cfg) + Expect(kubeClientErr).NotTo(HaveOccurred()) + + // Create istio-system namespace + _, meshNamespace := utils.GetIstioControlPlaneName(ctx, k8sClient) + istioNamespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: meshNamespace, + Namespace: meshNamespace, + }, + } + Expect(k8sClient.Create(ctx, istioNamespace)).Should(Succeed()) + + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme.Scheme, + Client: client.Options{ + Cache: &client.CacheOptions{ + DisableFor: []client.Object{&istiov1beta1.AuthorizationPolicy{}}, + }, + }, + Cache: cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + &corev1.Secret{}: { + Label: k8sLabels.SelectorFromSet(k8sLabels.Set{ + "opendatahub.io/managed": "true", + }), + }, + }, + }, + Metrics: metricsserver.Options{BindAddress: "0"}, + }) + Expect(err).NotTo(HaveOccurred()) + + err = NewInferenceServiceReconciler( + ctrl.Log.WithName("setup"), + mgr.GetClient(), + mgr.GetScheme(), + mgr.GetAPIReader(), + kubeClient, + false, + true, + false, + "", + ).SetupWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + + err = (&ServingRuntimeReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + MonitoringNS: MonitoringNS, + }).SetupWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + + go func() { + defer GinkgoRecover() + err = mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred(), "failed to start manager") + }() +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + cancel() + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) + +// Cleanup resources to not contaminate between tests +var _ = AfterEach(func() { + cleanUp := func(namespace string, cli client.Client) { + inNamespace := client.InNamespace(namespace) + _, meshNamespace := utils.GetIstioControlPlaneName(ctx, cli) + istioNamespace := client.InNamespace(meshNamespace) + Expect(cli.DeleteAllOf(context.TODO(), &kservev1alpha1.ServingRuntime{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &kservev1beta1.InferenceService{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &routev1.Route{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &monitoringv1.ServiceMonitor{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &k8srbacv1.RoleBinding{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &corev1.Secret{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &authorinov1beta2.AuthConfig{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &corev1.ConfigMap{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &corev1.Service{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &istioclientv1beta1.Gateway{}, istioNamespace)).ToNot(HaveOccurred()) + } + cleanUp(WorkingNamespace, k8sClient) + for _, ns := range testutils.Namespaces.All() { + cleanUp(ns, k8sClient) + } + testutils.Namespaces.Clear() +}) diff --git a/controllers/testdata/configmaps/inferenceservice-config.yaml b/internal/controller/serving/testdata/configmaps/inferenceservice-config.yaml similarity index 100% rename from controllers/testdata/configmaps/inferenceservice-config.yaml rename to internal/controller/serving/testdata/configmaps/inferenceservice-config.yaml diff --git a/controllers/testdata/deploy/kserve-nil-model-inference-service.yaml b/internal/controller/serving/testdata/deploy/kserve-nil-model-inference-service.yaml similarity index 100% rename from controllers/testdata/deploy/kserve-nil-model-inference-service.yaml rename to internal/controller/serving/testdata/deploy/kserve-nil-model-inference-service.yaml diff --git a/controllers/testdata/deploy/kserve-nil-runtime-inference-service.yaml b/internal/controller/serving/testdata/deploy/kserve-nil-runtime-inference-service.yaml similarity index 100% rename from controllers/testdata/deploy/kserve-nil-runtime-inference-service.yaml rename to internal/controller/serving/testdata/deploy/kserve-nil-runtime-inference-service.yaml diff --git a/controllers/testdata/deploy/kserve-openvino-inference-service-1.yaml b/internal/controller/serving/testdata/deploy/kserve-openvino-inference-service-1.yaml similarity index 100% rename from controllers/testdata/deploy/kserve-openvino-inference-service-1.yaml rename to internal/controller/serving/testdata/deploy/kserve-openvino-inference-service-1.yaml diff --git a/controllers/testdata/deploy/kserve-openvino-serving-runtime-1.yaml b/internal/controller/serving/testdata/deploy/kserve-openvino-serving-runtime-1.yaml similarity index 100% rename from controllers/testdata/deploy/kserve-openvino-serving-runtime-1.yaml rename to internal/controller/serving/testdata/deploy/kserve-openvino-serving-runtime-1.yaml diff --git a/controllers/testdata/deploy/kserve-unsupported-metrics-inference-service.yaml b/internal/controller/serving/testdata/deploy/kserve-unsupported-metrics-inference-service.yaml similarity index 100% rename from controllers/testdata/deploy/kserve-unsupported-metrics-inference-service.yaml rename to internal/controller/serving/testdata/deploy/kserve-unsupported-metrics-inference-service.yaml diff --git a/controllers/testdata/deploy/kserve-unsupported-metrics-serving-runtime.yaml b/internal/controller/serving/testdata/deploy/kserve-unsupported-metrics-serving-runtime.yaml similarity index 100% rename from controllers/testdata/deploy/kserve-unsupported-metrics-serving-runtime.yaml rename to internal/controller/serving/testdata/deploy/kserve-unsupported-metrics-serving-runtime.yaml diff --git a/controllers/testdata/deploy/openvino-inference-service-1.yaml b/internal/controller/serving/testdata/deploy/openvino-inference-service-1.yaml similarity index 100% rename from controllers/testdata/deploy/openvino-inference-service-1.yaml rename to internal/controller/serving/testdata/deploy/openvino-inference-service-1.yaml diff --git a/controllers/testdata/deploy/openvino-inference-service-no-runtime.yaml b/internal/controller/serving/testdata/deploy/openvino-inference-service-no-runtime.yaml similarity index 100% rename from controllers/testdata/deploy/openvino-inference-service-no-runtime.yaml rename to internal/controller/serving/testdata/deploy/openvino-inference-service-no-runtime.yaml diff --git a/controllers/testdata/deploy/test-openvino-serving-runtime-1.yaml b/internal/controller/serving/testdata/deploy/test-openvino-serving-runtime-1.yaml similarity index 100% rename from controllers/testdata/deploy/test-openvino-serving-runtime-1.yaml rename to internal/controller/serving/testdata/deploy/test-openvino-serving-runtime-1.yaml diff --git a/controllers/testdata/deploy/test-openvino-serving-runtime-2.yaml b/internal/controller/serving/testdata/deploy/test-openvino-serving-runtime-2.yaml similarity index 100% rename from controllers/testdata/deploy/test-openvino-serving-runtime-2.yaml rename to internal/controller/serving/testdata/deploy/test-openvino-serving-runtime-2.yaml diff --git a/controllers/testdata/dsci-with-authorino-missing.yaml b/internal/controller/serving/testdata/dsci-with-authorino-missing.yaml similarity index 100% rename from controllers/testdata/dsci-with-authorino-missing.yaml rename to internal/controller/serving/testdata/dsci-with-authorino-missing.yaml diff --git a/controllers/testdata/gateway/kserve-local-gateway.yaml b/internal/controller/serving/testdata/gateway/kserve-local-gateway.yaml similarity index 100% rename from controllers/testdata/gateway/kserve-local-gateway.yaml rename to internal/controller/serving/testdata/gateway/kserve-local-gateway.yaml diff --git a/controllers/testdata/gateway/test-isvc-svc-secret.yaml b/internal/controller/serving/testdata/gateway/test-isvc-svc-secret.yaml similarity index 100% rename from controllers/testdata/gateway/test-isvc-svc-secret.yaml rename to internal/controller/serving/testdata/gateway/test-isvc-svc-secret.yaml diff --git a/controllers/testdata/kserve-authorization-policy.yaml b/internal/controller/serving/testdata/kserve-authorization-policy.yaml similarity index 100% rename from controllers/testdata/kserve-authorization-policy.yaml rename to internal/controller/serving/testdata/kserve-authorization-policy.yaml diff --git a/controllers/testdata/results/example-onnx-mnist-no-runtime-route.yaml b/internal/controller/serving/testdata/results/example-onnx-mnist-no-runtime-route.yaml similarity index 100% rename from controllers/testdata/results/example-onnx-mnist-no-runtime-route.yaml rename to internal/controller/serving/testdata/results/example-onnx-mnist-no-runtime-route.yaml diff --git a/controllers/testdata/results/example-onnx-mnist-route.yaml b/internal/controller/serving/testdata/results/example-onnx-mnist-route.yaml similarity index 100% rename from controllers/testdata/results/example-onnx-mnist-route.yaml rename to internal/controller/serving/testdata/results/example-onnx-mnist-route.yaml diff --git a/controllers/testdata/results/model-server-ns-role.yaml b/internal/controller/serving/testdata/results/model-server-ns-role.yaml similarity index 100% rename from controllers/testdata/results/model-server-ns-role.yaml rename to internal/controller/serving/testdata/results/model-server-ns-role.yaml diff --git a/controllers/testdata/servingcert-service/test-isvc-svc.yaml b/internal/controller/serving/testdata/servingcert-service/test-isvc-svc.yaml similarity index 100% rename from controllers/testdata/servingcert-service/test-isvc-svc.yaml rename to internal/controller/serving/testdata/servingcert-service/test-isvc-svc.yaml diff --git a/controllers/testdata/dsci-with-authorino-enabled.yaml b/internal/controller/testdata/dsci-with-authorino-enabled.yaml similarity index 100% rename from controllers/testdata/dsci-with-authorino-enabled.yaml rename to internal/controller/testdata/dsci-with-authorino-enabled.yaml diff --git a/controllers/testdata/nvidia_api_mock.go b/internal/controller/testdata/nvidia_api_mock.go similarity index 85% rename from controllers/testdata/nvidia_api_mock.go rename to internal/controller/testdata/nvidia_api_mock.go index e1a8b8a4..988eb862 100644 --- a/controllers/testdata/nvidia_api_mock.go +++ b/internal/controller/testdata/nvidia_api_mock.go @@ -21,11 +21,12 @@ import ( "encoding/json" "errors" "fmt" - "github.com/opendatahub-io/odh-model-controller/controllers/utils" "io" "net/http" "os" "strings" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" ) type NimHttpClientMock struct{} @@ -41,7 +42,7 @@ func (r *NimHttpClientMock) Do(req *http.Request) (*http.Response, error) { catParams := &utils.NimCatalogQuery{} _ = json.Unmarshal([]byte(req.URL.Query().Get("q")), catParams) if catParams.Query == "orgName:nim" { - f, _ := os.ReadFile(fmt.Sprintf("testdata/nim/ngc_catalog_response_page_%d.json", catParams.Page)) + f, _ := os.ReadFile(fmt.Sprintf("testdata/ngc_catalog_response_page_%d.json", catParams.Page)) return &http.Response{StatusCode: 200, Body: io.NopCloser(bytes.NewReader(f))}, nil } } @@ -51,11 +52,11 @@ func (r *NimHttpClientMock) Do(req *http.Request) (*http.Response, error) { if req.URL.Query().Get("account") == "$oauthtoken" && req.URL.Query().Get("offline_token") == "true" { if req.URL.Query().Get("scope") == "repository:nim/microsoft/phi-3-mini-4k-instruct:pull" { // repository name "nim/microsoft/phi-3-mini-4k-instruct" is the FIRST resource from the available - // from runtimes returned by the ngc catalog endpoint, check testdata/nim/ngc_catalog_response_page_0.json + // from runtimes returned by the ngc catalog endpoint, check testdata/ngc_catalog_response_page_0.json authHeaderParts := strings.Split(req.Header.Get("Authorization"), " ") token, _ := base64.StdEncoding.DecodeString(authHeaderParts[1]) if authHeaderParts[0] == "Basic" && string(token) == "$oauthtoken:"+FakeApiKey { - f, _ := os.ReadFile("testdata/nim/runtime_token_response.json") + f, _ := os.ReadFile("testdata/runtime_token_response.json") return &http.Response{StatusCode: 200, Body: io.NopCloser(bytes.NewReader(f))}, nil } } @@ -66,10 +67,10 @@ func (r *NimHttpClientMock) Do(req *http.Request) (*http.Response, error) { if req.URL.Host == "nvcr.io" && req.URL.Path == "/v2/nim/microsoft/phi-3-mini-4k-instruct/manifests/1.2.3" { // repository name "nim/microsoft/phi-3-mini-4k-instruct" is the FIRST resource from the available // from runtimes returned by the ngc catalog endpoint, version "1.2.3" is the latestTag attribute for the runtime - // check testdata/nim/ngc_catalog_response_page_0.json + // check testdata/ngc_catalog_response_page_0.json authHeaderParts := strings.Split(req.Header.Get("Authorization"), " ") if authHeaderParts[0] == "Bearer" && authHeaderParts[1] == "this-is-my-fake-token-please-dont-share-it-with-anyone" { - // the token is returned by the nvcr.io/proxy-auth endpoint (stubbed), check testdata/nim/runtime_token_response.json + // the token is returned by the nvcr.io/proxy-auth endpoint (stubbed), check testdata/runtime_token_response.json return &http.Response{StatusCode: 200}, nil } } @@ -78,7 +79,7 @@ func (r *NimHttpClientMock) Do(req *http.Request) (*http.Response, error) { if req.URL.Host == "authn.nvidia.com" && req.URL.Path == "/token" && req.URL.Query().Get("service") == "ngc" { authHeaderParts := strings.Split(req.Header.Get("Authorization"), " ") if authHeaderParts[0] == "ApiKey" && authHeaderParts[1] == FakeApiKey { - f, _ := os.ReadFile("testdata/nim/ngc_token_response.json") + f, _ := os.ReadFile("testdata/ngc_token_response.json") return &http.Response{StatusCode: 200, Body: io.NopCloser(bytes.NewReader(f))}, nil } } @@ -86,11 +87,11 @@ func (r *NimHttpClientMock) Do(req *http.Request) (*http.Response, error) { // stub model info for the FIRST model we stub, requested by utils.GetNimModelData (nim) if req.URL.Host == "api.ngc.nvidia.com" && req.URL.Path == "/v2/org/nim/team/microsoft/repos/phi-3-mini-4k-instruct" { // repository name "nim/microsoft/phi-3-mini-4k-instruct" is the FIRST resource from the available - // from runtimes returned by the ngc catalog endpoint, check testdata/nim/ngc_catalog_response_page_0.json + // from runtimes returned by the ngc catalog endpoint, check testdata/ngc_catalog_response_page_0.json authHeaderParts := strings.Split(req.Header.Get("Authorization"), " ") if authHeaderParts[0] == "Bearer" && authHeaderParts[1] == "this-is-yet-another-fake-token-of-mine-you-know-what-not-do-to" { - // the token is returned by the authn.nvidia.com/token endpoint (stubbed), check testdata/nim/ngc_token_response.json - f, _ := os.ReadFile("testdata/nim/ngc_model_phi-3-mini-4k-instruct_response.json") + // the token is returned by the authn.nvidia.com/token endpoint (stubbed), check testdata/ngc_token_response.json + f, _ := os.ReadFile("testdata/ngc_model_phi-3-mini-4k-instruct_response.json") return &http.Response{StatusCode: 200, Body: io.NopCloser(bytes.NewReader(f))}, nil } } @@ -98,11 +99,11 @@ func (r *NimHttpClientMock) Do(req *http.Request) (*http.Response, error) { // stub model info for the SECOND model we stub, requested by utils.GetNimModelData (nim) if req.URL.Host == "api.ngc.nvidia.com" && req.URL.Path == "/v2/org/nim/team/meta/repos/llama-3.1-8b-instruct" { // repository name "nim/meta/llama-3.1-8b-instruct" is the SECOND resource from the available - // from runtimes returned by the ngc catalog endpoint, check testdata/nim/ngc_catalog_response_page_1.json + // from runtimes returned by the ngc catalog endpoint, check testdata/ngc_catalog_response_page_1.json authHeaderParts := strings.Split(req.Header.Get("Authorization"), " ") if authHeaderParts[0] == "Bearer" && authHeaderParts[1] == "this-is-yet-another-fake-token-of-mine-you-know-what-not-do-to" { - // the token is returned by the authn.nvidia.com/token endpoint (stubbed), check testdata/nim/ngc_token_response.json - f, _ := os.ReadFile("testdata/nim/ngc_model_llama-3_1-8b-instruct_response.json") + // the token is returned by the authn.nvidia.com/token endpoint (stubbed), check testdata/ngc_token_response.json + f, _ := os.ReadFile("testdata/ngc_model_llama-3_1-8b-instruct_response.json") return &http.Response{StatusCode: 200, Body: io.NopCloser(bytes.NewReader(f))}, nil } } diff --git a/controllers/testdata/results/iris-unsupported-metrics-dashboard.yaml b/internal/controller/testdata/results/iris-unsupported-metrics-dashboard.yaml similarity index 100% rename from controllers/testdata/results/iris-unsupported-metrics-dashboard.yaml rename to internal/controller/testdata/results/iris-unsupported-metrics-dashboard.yaml diff --git a/controllers/testdata/results/mnist-ovms-metrics-dashboard.yaml b/internal/controller/testdata/results/mnist-ovms-metrics-dashboard.yaml similarity index 100% rename from controllers/testdata/results/mnist-ovms-metrics-dashboard.yaml rename to internal/controller/testdata/results/mnist-ovms-metrics-dashboard.yaml diff --git a/controllers/testdata/secrets/dataconnection-encoded.yaml b/internal/controller/testdata/secrets/dataconnection-encoded.yaml similarity index 100% rename from controllers/testdata/secrets/dataconnection-encoded.yaml rename to internal/controller/testdata/secrets/dataconnection-encoded.yaml diff --git a/controllers/testdata/secrets/storageconfig-string-unmanaged.yaml b/internal/controller/testdata/secrets/storageconfig-string-unmanaged.yaml similarity index 100% rename from controllers/testdata/secrets/storageconfig-string-unmanaged.yaml rename to internal/controller/testdata/secrets/storageconfig-string-unmanaged.yaml diff --git a/controllers/testdata/secrets/storageconfig-string.yaml b/internal/controller/testdata/secrets/storageconfig-string.yaml similarity index 100% rename from controllers/testdata/secrets/storageconfig-string.yaml rename to internal/controller/testdata/secrets/storageconfig-string.yaml diff --git a/controllers/testdata/secrets/storageconfig-updated-cert-string.yaml b/internal/controller/testdata/secrets/storageconfig-updated-cert-string.yaml similarity index 100% rename from controllers/testdata/secrets/storageconfig-updated-cert-string.yaml rename to internal/controller/testdata/secrets/storageconfig-updated-cert-string.yaml diff --git a/controllers/utils/conditions.go b/internal/controller/utils/conditions.go similarity index 99% rename from controllers/utils/conditions.go rename to internal/controller/utils/conditions.go index abe8a8e1..17f79f20 100644 --- a/controllers/utils/conditions.go +++ b/internal/controller/utils/conditions.go @@ -16,8 +16,9 @@ limitations under the License. package utils import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) type NimConditionType string diff --git a/controllers/utils/converter.go b/internal/controller/utils/converter.go similarity index 100% rename from controllers/utils/converter.go rename to internal/controller/utils/converter.go diff --git a/controllers/utils/gvk.go b/internal/controller/utils/gvk.go similarity index 100% rename from controllers/utils/gvk.go rename to internal/controller/utils/gvk.go diff --git a/controllers/utils/init.go b/internal/controller/utils/init.go similarity index 95% rename from controllers/utils/init.go rename to internal/controller/utils/init.go index 184efe8c..5210811e 100644 --- a/controllers/utils/init.go +++ b/internal/controller/utils/init.go @@ -4,7 +4,6 @@ import ( kservev1alpha1 "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" authorinov1beta2 "github.com/kuadrant/authorino/api/v1beta2" - nimv1 "github.com/opendatahub-io/odh-model-controller/api/nim/v1" routev1 "github.com/openshift/api/route/v1" templatev1 "github.com/openshift/api/template/v1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" @@ -17,6 +16,8 @@ import ( knservingv1 "knative.dev/serving/pkg/apis/serving/v1" maistrav1 "maistra.io/api/core/v1" + nimv1 "github.com/opendatahub-io/odh-model-controller/api/nim/v1" + "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" ) @@ -43,7 +44,7 @@ func RegisterSchemes(s *runtime.Scheme) { // The following are related to Service Mesh, uncomment this and other // similar blocks to use with Service Mesh - //utilruntime.Must(virtualservicev1.AddToScheme(scheme)) + // utilruntime.Must(virtualservicev1.AddToScheme(scheme)) - //+kubebuilder:scaffold:scheme + // +kubebuilder:scaffold:scheme } diff --git a/controllers/utils/nim.go b/internal/controller/utils/nim.go similarity index 99% rename from controllers/utils/nim.go rename to internal/controller/utils/nim.go index 4243ab4b..2c2d1a77 100644 --- a/controllers/utils/nim.go +++ b/internal/controller/utils/nim.go @@ -19,17 +19,18 @@ import ( "encoding/base64" "encoding/json" "fmt" + "io" + "net/http" + "strings" + "time" + "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" kserveconstants "github.com/kserve/kserve/pkg/constants" - "io" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "net/http" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" - "strings" - "time" ) type ( diff --git a/controllers/utils/utils.go b/internal/controller/utils/utils.go similarity index 96% rename from controllers/utils/utils.go rename to internal/controller/utils/utils.go index 7fde9682..e3047a39 100644 --- a/controllers/utils/utils.go +++ b/internal/controller/utils/utils.go @@ -11,12 +11,10 @@ import ( "github.com/go-logr/logr" kservev1alpha1 "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" - "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/types" - kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" "github.com/kuadrant/authorino/pkg/log" - "github.com/opendatahub-io/odh-model-controller/controllers/constants" + operatorv1 "github.com/openshift/api/operator/v1" + "github.com/pkg/errors" v1beta12 "istio.io/api/security/v1beta1" "istio.io/client-go/pkg/apis/security/v1beta1" corev1 "k8s.io/api/core/v1" @@ -25,9 +23,12 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/discovery" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" ) var ( @@ -119,7 +120,7 @@ func VerifyIfComponentIsEnabled(ctx context.Context, cli client.Client, componen componentName, objectList.Items[0], errKserve) } - return (serving == "Managed" || serving == "Unmanaged") && kserve == "Managed", nil + return (serving == string(operatorv1.Managed) || serving == string(operatorv1.Unmanaged)) && kserve == string(operatorv1.Managed), nil } val, _, err := unstructured.NestedString(objectList.Items[0].Object, fields...) @@ -127,7 +128,7 @@ func VerifyIfComponentIsEnabled(ctx context.Context, cli client.Client, componen return false, fmt.Errorf("failed to retrieve the component [%s] status from %+v", componentName, objectList.Items[0]) } - return val == "Managed", nil + return val == string(operatorv1.Managed), nil } else { return false, fmt.Errorf("there is no %s available in the cluster", GVK.DataScienceCluster.Kind) } @@ -400,16 +401,16 @@ func FindSupportingRuntimeForISvc(ctx context.Context, cli client.Client, log lo }) for _, runtime := range runtimes.Items { - if runtime.Spec.Disabled != nil && *runtime.Spec.Disabled == true { + if runtime.Spec.Disabled != nil && *runtime.Spec.Disabled { continue } - if runtime.Spec.MultiModel != nil && *runtime.Spec.MultiModel == false { + if runtime.Spec.MultiModel != nil && !*runtime.Spec.MultiModel { continue } for _, supportedFormat := range runtime.Spec.SupportedModelFormats { - if supportedFormat.AutoSelect != nil && *supportedFormat.AutoSelect == true && supportedFormat.Name == isvc.Spec.Predictor.Model.ModelFormat.Name { + if supportedFormat.AutoSelect != nil && *supportedFormat.AutoSelect && supportedFormat.Name == isvc.Spec.Predictor.Model.ModelFormat.Name { desiredServingRuntime = &runtime log.Info("Automatic runtime selection for InferenceService", "runtime", desiredServingRuntime.Name) return desiredServingRuntime, nil diff --git a/internal/webhook/nim/v1/account_webhook.go b/internal/webhook/nim/v1/account_webhook.go new file mode 100644 index 00000000..25b216a1 --- /dev/null +++ b/internal/webhook/nim/v1/account_webhook.go @@ -0,0 +1,106 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "context" + "fmt" + + "sigs.k8s.io/controller-runtime/pkg/client" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + nimv1 "github.com/opendatahub-io/odh-model-controller/api/nim/v1" +) + +// nolint:unused +// log is for logging in this package. +var accountlog = logf.Log.WithName("NIMAccountValidatingWebhook") + +// SetupAccountWebhookWithManager registers the webhook for Account in the manager. +func SetupAccountWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr).For(&nimv1.Account{}). + WithValidator(&AccountCustomValidator{client: mgr.GetClient()}). + Complete() +} + +// TODO(user): EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! + +// NOTE: The 'path' attribute must follow a specific pattern and should not be modified directly here. +// Modifying the path for an invalid path can cause API server errors; failing to locate the webhook. +// +kubebuilder:webhook:path=/validate-nim-opendatahub-io-v1-account,mutating=false,failurePolicy=fail,sideEffects=None,groups=nim.opendatahub.io,resources=accounts,verbs=create;update,versions=v1,name=validating.nim.account.odh-model-controller.opendatahub.io,admissionReviewVersions=v1 + +// AccountCustomValidator struct is responsible for validating the Account resource +// when it is created, updated, or deleted. +// +// NOTE: The +kubebuilder:object:generate=false marker prevents controller-gen from generating DeepCopy methods, +// as this struct is used only for temporary operations and does not need to be deeply copied. +type AccountCustomValidator struct { + client client.Client +} + +var _ webhook.CustomValidator = &AccountCustomValidator{} + +// ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type Account. +func (v *AccountCustomValidator) ValidateCreate(ctx context.Context, obj runtime.Object) (warnings admission.Warnings, err error) { + account, ok := obj.(*nimv1.Account) + if !ok { + return nil, fmt.Errorf("expected a Account object but got %T", obj) + } + + log := accountlog.WithValues("namespace", account.Namespace, "account", account.Name) + log.Info("Validating NIM Account creation") + + err = v.verifySingletonInNamespace(ctx, account) + if err != nil { + log.Error(err, "Rejecting NIM Account creation because checking singleton didn't pass") + return nil, err + } + + return nil, nil +} + +// ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type Account. +func (v *AccountCustomValidator) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { + // For update, nothing needs to be validated + return nil, nil +} + +// ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type Account. +func (v *AccountCustomValidator) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + // For deletion, nothing needs to be validated + return nil, nil +} + +func (v *AccountCustomValidator) verifySingletonInNamespace(ctx context.Context, account *nimv1.Account) error { + accountList := nimv1.AccountList{} + err := v.client.List(ctx, + &accountList, + client.InNamespace(account.Namespace)) + if err != nil { + return fmt.Errorf("failed to verify if there are existing Accounts with err: %s", err.Error()) + } + + if len(accountList.Items) > 0 { + return fmt.Errorf("rejecting creation of Account %s in namespace %s because there is already an Account created in the namespace", account.Name, account.Namespace) + } + return nil +} diff --git a/controllers/webhook_nim_account_validator_test.go b/internal/webhook/nim/v1/account_webhook_test.go similarity index 62% rename from controllers/webhook_nim_account_validator_test.go rename to internal/webhook/nim/v1/account_webhook_test.go index 7d3042ba..f5bdc40b 100644 --- a/controllers/webhook_nim_account_validator_test.go +++ b/internal/webhook/nim/v1/account_webhook_test.go @@ -1,21 +1,36 @@ -package controllers +/* +Copyright 2024. -import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at - nimv1 "github.com/opendatahub-io/odh-model-controller/api/nim/v1" - "github.com/opendatahub-io/odh-model-controller/controllers/webhook" + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + testutils "github.com/opendatahub-io/odh-model-controller/test/utils" + + nimv1 "github.com/opendatahub-io/odh-model-controller/api/nim/v1" ) var _ = Describe("NIM Account validator webhook", func() { - var validator admission.CustomValidator + var validator AccountCustomValidator var nimNamespace *v1.Namespace var secretName = "nim-api-key-secret" @@ -35,14 +50,14 @@ var _ = Describe("NIM Account validator webhook", func() { }, }, } - Expect(cli.Create(ctx, account)).To(Succeed()) + Expect(k8sClient.Create(ctx, account)).To(Succeed()) Eventually(func() bool { - if err := cli.Get(ctx, client.ObjectKeyFromObject(account), account); err != nil { + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(account), account); err != nil { return false } return true - }, timeout, interval).Should(BeTrue()) + }, testTimeout, testInterval).Should(BeTrue()) } } @@ -54,21 +69,21 @@ var _ = Describe("NIM Account validator webhook", func() { Namespace: nimNamespace.Name, }, } - Expect(cli.Delete(ctx, account)).To(Succeed()) + Expect(k8sClient.Delete(ctx, account)).To(Succeed()) Eventually(func() bool { - err := cli.Get(ctx, client.ObjectKeyFromObject(account), account) + err := k8sClient.Get(ctx, client.ObjectKeyFromObject(account), account) if err != nil && errors.IsNotFound(err) { return true } return false - }, timeout, interval).Should(BeTrue()) + }, testTimeout, testInterval).Should(BeTrue()) } } BeforeEach(func() { - nimNamespace = Namespaces.Create(cli) - validator = webhook.NewNimAccountValidator(cli) + nimNamespace = testutils.Namespaces.Create(ctx, k8sClient) + validator = AccountCustomValidator{k8sClient} }) Context("when there is an existing NIM Account", func() { diff --git a/internal/webhook/nim/v1/webhook_suite_test.go b/internal/webhook/nim/v1/webhook_suite_test.go new file mode 100644 index 00000000..c8e117c1 --- /dev/null +++ b/internal/webhook/nim/v1/webhook_suite_test.go @@ -0,0 +1,187 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "path/filepath" + "runtime" + "testing" + "time" + + kservev1alpha1 "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" + kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" + authorinov1beta2 "github.com/kuadrant/authorino/api/v1beta2" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + routev1 "github.com/openshift/api/route/v1" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + istioclientv1beta1 "istio.io/client-go/pkg/apis/networking/v1beta1" + admissionv1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + k8srbacv1 "k8s.io/api/rbac/v1" + apimachineryruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + // +kubebuilder:scaffold:imports + + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" + testutils "github.com/opendatahub-io/odh-model-controller/test/utils" +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +// TODO: Deduplicate +const testTimeout = time.Second * 20 +const testInterval = time.Millisecond * 10 + +var ( + cancel context.CancelFunc + cfg *rest.Config + ctx context.Context + k8sClient client.Client + testEnv *envtest.Environment +) + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Webhook Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{ + filepath.Join("..", "..", "..", "..", "config", "crd", "bases"), + filepath.Join("..", "..", "..", "..", "config", "crd", "external")}, + ErrorIfCRDPathMissing: true, + + // The BinaryAssetsDirectory is only required if you want to run the tests directly + // without call the makefile target test. If not informed it will look for the + // default path defined in controller-runtime which is /usr/local/kubebuilder/. + // Note that you must have the required binaries setup under the bin directory to perform + // the tests directly. When we run make test it will be setup and used automatically. + BinaryAssetsDirectory: filepath.Join("..", "..", "..", "..", "bin", "k8s", + fmt.Sprintf("1.31.0-%s-%s", runtime.GOOS, runtime.GOARCH)), + + WebhookInstallOptions: envtest.WebhookInstallOptions{ + Paths: []string{filepath.Join("..", "..", "..", "..", "config", "webhook")}, + }, + } + + var err error + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + scheme := apimachineryruntime.NewScheme() + utils.RegisterSchemes(scheme) + + err = admissionv1.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + + // +kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + // start webhook server using Manager. + webhookInstallOptions := &testEnv.WebhookInstallOptions + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme, + WebhookServer: webhook.NewServer(webhook.Options{ + Host: webhookInstallOptions.LocalServingHost, + Port: webhookInstallOptions.LocalServingPort, + CertDir: webhookInstallOptions.LocalServingCertDir, + }), + LeaderElection: false, + Metrics: metricsserver.Options{BindAddress: "0"}, + }) + Expect(err).NotTo(HaveOccurred()) + + err = SetupAccountWebhookWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + + // +kubebuilder:scaffold:webhook + + go func() { + defer GinkgoRecover() + err = mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred()) + }() + + // wait for the webhook server to get ready. + dialer := &net.Dialer{Timeout: time.Second} + addrPort := fmt.Sprintf("%s:%d", webhookInstallOptions.LocalServingHost, webhookInstallOptions.LocalServingPort) + Eventually(func() error { + conn, err := tls.DialWithDialer(dialer, "tcp", addrPort, &tls.Config{InsecureSkipVerify: true}) + if err != nil { + return err + } + + return conn.Close() + }).Should(Succeed()) +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + cancel() + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) + +// TODO: Deduplicate (if needed) +// Cleanup resources to not contaminate between tests +var _ = AfterEach(func() { + cleanUp := func(namespace string, cli client.Client) { + inNamespace := client.InNamespace(namespace) + _, meshNamespace := utils.GetIstioControlPlaneName(ctx, cli) + istioNamespace := client.InNamespace(meshNamespace) + Expect(cli.DeleteAllOf(context.TODO(), &kservev1alpha1.ServingRuntime{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &kservev1beta1.InferenceService{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &routev1.Route{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &monitoringv1.ServiceMonitor{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &k8srbacv1.RoleBinding{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &corev1.Secret{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &authorinov1beta2.AuthConfig{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &corev1.ConfigMap{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &corev1.Service{}, inNamespace)).ToNot(HaveOccurred()) + Expect(cli.DeleteAllOf(context.TODO(), &istioclientv1beta1.Gateway{}, istioNamespace)).ToNot(HaveOccurred()) + } + for _, ns := range testutils.Namespaces.All() { + cleanUp(ns, k8sClient) + } + testutils.Namespaces.Clear() +}) diff --git a/internal/webhook/serving/v1/service_webhook.go b/internal/webhook/serving/v1/service_webhook.go new file mode 100644 index 00000000..cb2d7ef5 --- /dev/null +++ b/internal/webhook/serving/v1/service_webhook.go @@ -0,0 +1,148 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "context" + "fmt" + "strings" + + kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" + "k8s.io/apimachinery/pkg/runtime" + types2 "k8s.io/apimachinery/pkg/types" + servingv1 "knative.dev/serving/pkg/apis/serving/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" + "github.com/opendatahub-io/odh-model-controller/internal/controller/resources" + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" +) + +// nolint:unused +// log is for logging in this package. +var servicelog = logf.Log.WithName("knative-service-validating-webhook") + +// SetupServiceWebhookWithManager registers the webhook for Service in the manager. +func SetupServiceWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr).For(&servingv1.Service{}). + WithValidator(&ServiceCustomValidator{client: mgr.GetClient()}). + Complete() +} + +// NOTE: The 'path' attribute must follow a specific pattern and should not be modified directly here. +// Modifying the path for an invalid path can cause API server errors; failing to locate the webhook. +// +kubebuilder:webhook:path=/validate-serving-knative-dev-v1-service,mutating=false,failurePolicy=fail,sideEffects=None,groups="serving.knative.dev",resources=services,verbs=create,versions=v1,name=validating.ksvc.odh-model-controller.opendatahub.io,admissionReviewVersions=v1 + +// ServiceCustomValidator struct is responsible for validating the Knative Service resource +// when it is created, updated, or deleted. +// +// NOTE: The +kubebuilder:object:generate=false marker prevents controller-gen from generating DeepCopy methods, +// as this struct is used only for temporary operations and does not need to be deeply copied. +type ServiceCustomValidator struct { + client client.Client +} + +var _ webhook.CustomValidator = &ServiceCustomValidator{} + +// ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the Knative type Service. +func (v *ServiceCustomValidator) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + ksvc, ok := obj.(*servingv1.Service) + if !ok { + return nil, fmt.Errorf("expected a Knative Service object but got %T", obj) + } + + logger := servicelog.WithValues("namespace", ksvc.Namespace, "ksvc", ksvc.Name) + logger.Info("Validation for Knative Service upon creation") + + // If there is an explicit intent for not having a sidecar, skip validation + ksvcTemplateMeta := ksvc.Spec.Template.GetObjectMeta() + // if ksvcTemplateMeta != nil { + if templateAnnotations := ksvcTemplateMeta.GetAnnotations(); templateAnnotations[constants.IstioSidecarInjectAnnotationName] == "false" { + logger.V(1).Info("Skipping validation of Knative Service because there is an explicit intent to exclude it from the Service Mesh") + return nil, nil + } + // } + + // Only validate the KSVC if it is owned by KServe controller + ksvcMetadata := ksvc.GetObjectMeta() + // if ksvcMetadata == nil { + // logger.V(1).Info("Skipping validation of Knative Service because it does not have metadata") + // return nil, nil + // } + ksvcOwnerReferences := ksvcMetadata.GetOwnerReferences() + if ksvcOwnerReferences == nil { + logger.V(1).Info("Skipping validation of Knative Service because it does not have owner references") + return nil, nil + } + isOwnedByKServe := false + for _, owner := range ksvcOwnerReferences { + if owner.Kind == constants.InferenceServiceKind { + if strings.Contains(owner.APIVersion, kservev1beta1.SchemeGroupVersion.Group) { + isOwnedByKServe = true + } + } + } + if !isOwnedByKServe { + logger.V(1).Info("Skipping validation of Knative Service because it is not owned by KServe") + return nil, nil + } + + // Since the Ksvc is owned by an InferenceService, it is known that it is required to be + // in the Mesh. Thus, the involved namespace needs to be enrolled in the mesh. + // Go and check the ServiceMeshMemberRoll to verify that the namespace is already a + // member. If it is still not a member, reject creation of the Ksvc to prevent + // creation of a Pod that would not be in the Mesh. + smmrQuerier := resources.NewServiceMeshMemberRole(v.client) + _, meshNamespace := utils.GetIstioControlPlaneName(ctx, v.client) + smmr, fetchSmmrErr := smmrQuerier.FetchSMMR(ctx, logger, types2.NamespacedName{Name: constants.ServiceMeshMemberRollName, Namespace: meshNamespace}) + if fetchSmmrErr != nil { + logger.Error(fetchSmmrErr, "Error when fetching ServiceMeshMemberRoll", "smmr.namespace", meshNamespace, "smmr.name", constants.ServiceMeshMemberRollName) + return nil, fetchSmmrErr + } + if smmr == nil { + logger.Info("Rejecting Knative service because the ServiceMeshMemberRoll does not exist", "smmr.namespace", meshNamespace, "smmr.name", constants.ServiceMeshMemberRollName) + return nil, fmt.Errorf("rejecting creation of Knative service %s on namespace %s because the ServiceMeshMemberRoll does not exist", ksvc.Name, ksvc.Namespace) + } + + logger = logger.WithValues("smmr.namespace", smmr.Namespace, "smmr.name", smmr.Name) + + for _, memberNamespace := range smmr.Status.ConfiguredMembers { + if memberNamespace == ksvc.Namespace { + logger.V(1).Info("The Knative service is accepted") + return nil, nil + } + } + + logger.Info("Rejecting Knative service because its namespace is not a member of the service mesh") + return nil, fmt.Errorf("rejecting creation of Knative service %s on namespace %s because the namespace is not a configured member of the mesh", ksvc.Name, ksvc.Namespace) +} + +// ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the Knative type Service. +func (v *ServiceCustomValidator) ValidateUpdate(_ context.Context, _, _ runtime.Object) (admission.Warnings, error) { + // Nothing to validate on updates + return nil, nil +} + +// ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the Knative type Service. +func (v *ServiceCustomValidator) ValidateDelete(_ context.Context, _ runtime.Object) (admission.Warnings, error) { + // Nothing to validate on updates + return nil, nil +} diff --git a/controllers/webhook_ksvc_validator_test.go b/internal/webhook/serving/v1/service_webhook_test.go similarity index 86% rename from controllers/webhook_ksvc_validator_test.go rename to internal/webhook/serving/v1/service_webhook_test.go index b6670f4a..ab568495 100644 --- a/controllers/webhook_ksvc_validator_test.go +++ b/internal/webhook/serving/v1/service_webhook_test.go @@ -1,4 +1,5 @@ /* +Copyright 2024. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -13,14 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controllers +package v1 import ( kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/opendatahub-io/odh-model-controller/controllers/constants" - "github.com/opendatahub-io/odh-model-controller/controllers/utils" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -28,10 +27,11 @@ import ( v1 "maistra.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - "github.com/opendatahub-io/odh-model-controller/controllers/webhook" + "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" ) -var _ = Describe("Knative validator webhook", func() { +var _ = Describe("Knative Service Webhook", func() { var validator admission.CustomValidator var meshNamespace string @@ -60,22 +60,22 @@ var _ = Describe("Knative validator webhook", func() { }, } - Expect(cli.Create(ctx, smmr)).To(Succeed()) + Expect(k8sClient.Create(ctx, smmr)).To(Succeed()) smmr.Status = smmrStatus - Expect(cli.Status().Update(ctx, smmr)).To(Succeed()) + Expect(k8sClient.Status().Update(ctx, smmr)).To(Succeed()) return smmr } BeforeEach(func() { - _, meshNamespace = utils.GetIstioControlPlaneName(ctx, cli) - validator = webhook.NewKsvcValidator(cli) + _, meshNamespace = utils.GetIstioControlPlaneName(ctx, k8sClient) + validator = &ServiceCustomValidator{client: k8sClient} // Other tests may create a ServiceMeshMemberRoll. // If there is one, delete it because it conflicts with tests in this file. smmr := v1.ServiceMeshMemberRoll{} - getErr := cli.Get(ctx, types.NamespacedName{ + getErr := k8sClient.Get(ctx, types.NamespacedName{ Namespace: meshNamespace, Name: constants.ServiceMeshMemberRollName, }, &smmr) @@ -84,7 +84,7 @@ var _ = Describe("Knative validator webhook", func() { Fail("Error waiting for SMMR to be deleted: " + getErr.Error()) } } else { - cli.Delete(ctx, &smmr) + Expect(k8sClient.Delete(ctx, &smmr)).ShouldNot(HaveOccurred()) } }) @@ -148,7 +148,7 @@ var _ = Describe("Knative validator webhook", func() { It("should reject creating a Knative service if the ServiceMeshMemberRoll has null ConfiguredMembers", func() { ksvc := createKserveOwnedKsvc() smmr := createSmmr(v1.ServiceMeshMemberRollStatus{ConfiguredMembers: nil}) - defer func() { cli.Delete(ctx, smmr) }() + defer func() { _ = k8sClient.Delete(ctx, smmr) }() _, err := validator.ValidateCreate(ctx, ksvc) Expect(err).Should(HaveOccurred()) @@ -157,7 +157,7 @@ var _ = Describe("Knative validator webhook", func() { It("should reject creating a Knative service if the namespace is not a configured member of the ServiceMeshMemberRoll", func() { ksvc := createKserveOwnedKsvc() smmr := createSmmr(v1.ServiceMeshMemberRollStatus{ConfiguredMembers: []string{"foo"}}) - defer func() { cli.Delete(ctx, smmr) }() + defer func() { _ = k8sClient.Delete(ctx, smmr) }() _, err := validator.ValidateCreate(ctx, ksvc) Expect(err).Should(HaveOccurred()) @@ -166,7 +166,7 @@ var _ = Describe("Knative validator webhook", func() { It("should accept creating a Knative service if the namespace is a configured member of the ServiceMeshMemberRoll", func() { ksvc := createKserveOwnedKsvc() smmr := createSmmr(v1.ServiceMeshMemberRollStatus{ConfiguredMembers: []string{ksvc.Namespace}}) - defer func() { cli.Delete(ctx, smmr) }() + defer func() { _ = k8sClient.Delete(ctx, smmr) }() _, err := validator.ValidateCreate(ctx, ksvc) Expect(err).ShouldNot(HaveOccurred()) diff --git a/internal/webhook/serving/v1/webhook_suite_test.go b/internal/webhook/serving/v1/webhook_suite_test.go new file mode 100644 index 00000000..17bc8062 --- /dev/null +++ b/internal/webhook/serving/v1/webhook_suite_test.go @@ -0,0 +1,162 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "path/filepath" + "runtime" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + admissionv1 "k8s.io/api/admission/v1" + // +kubebuilder:scaffold:imports + apimachineryruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var ( + cancel context.CancelFunc + cfg *rest.Config + ctx context.Context + k8sClient client.Client + testEnv *envtest.Environment +) + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Webhook Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{ + filepath.Join("..", "..", "..", "..", "config", "crd", "bases"), + filepath.Join("..", "..", "..", "..", "config", "crd", "external")}, + ErrorIfCRDPathMissing: true, + + // The BinaryAssetsDirectory is only required if you want to run the tests directly + // without call the makefile target test. If not informed it will look for the + // default path defined in controller-runtime which is /usr/local/kubebuilder/. + // Note that you must have the required binaries setup under the bin directory to perform + // the tests directly. When we run make test it will be setup and used automatically. + BinaryAssetsDirectory: filepath.Join("..", "..", "..", "..", "bin", "k8s", + fmt.Sprintf("1.31.0-%s-%s", runtime.GOOS, runtime.GOARCH)), + + WebhookInstallOptions: envtest.WebhookInstallOptions{ + Paths: []string{filepath.Join("..", "..", "..", "..", "config", "webhook")}, + }, + } + + var err error + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + scheme := apimachineryruntime.NewScheme() + utils.RegisterSchemes(scheme) + + err = admissionv1.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + + // +kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + // Create istio-system namespace + _, meshNamespace := utils.GetIstioControlPlaneName(ctx, k8sClient) + istioNamespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: meshNamespace, + Namespace: meshNamespace, + }, + } + Expect(k8sClient.Create(ctx, istioNamespace)).Should(Succeed()) + + // start webhook server using Manager. + webhookInstallOptions := &testEnv.WebhookInstallOptions + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme, + WebhookServer: webhook.NewServer(webhook.Options{ + Host: webhookInstallOptions.LocalServingHost, + Port: webhookInstallOptions.LocalServingPort, + CertDir: webhookInstallOptions.LocalServingCertDir, + }), + LeaderElection: false, + Metrics: metricsserver.Options{BindAddress: "0"}, + }) + Expect(err).NotTo(HaveOccurred()) + + err = SetupServiceWebhookWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + + // +kubebuilder:scaffold:webhook + + go func() { + defer GinkgoRecover() + err = mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred()) + }() + + // wait for the webhook server to get ready. + dialer := &net.Dialer{Timeout: time.Second} + addrPort := fmt.Sprintf("%s:%d", webhookInstallOptions.LocalServingHost, webhookInstallOptions.LocalServingPort) + Eventually(func() error { + conn, err := tls.DialWithDialer(dialer, "tcp", addrPort, &tls.Config{InsecureSkipVerify: true}) + if err != nil { + return err + } + + return conn.Close() + }).Should(Succeed()) +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + cancel() + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/internal/webhook/serving/v1beta1/inferenceservice_webhook.go b/internal/webhook/serving/v1beta1/inferenceservice_webhook.go new file mode 100644 index 00000000..7ff082ce --- /dev/null +++ b/internal/webhook/serving/v1beta1/inferenceservice_webhook.go @@ -0,0 +1,129 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "context" + "fmt" + + servingv1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation/field" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" +) + +// nolint:unused +// log is for logging in this package. +var inferenceservicelog = logf.Log.WithName("inferenceservice-resource") + +// SetupInferenceServiceWebhookWithManager registers the webhook for InferenceService in the manager. +func SetupInferenceServiceWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr).For(&servingv1beta1.InferenceService{}). + WithValidator(&InferenceServiceCustomValidator{client: mgr.GetClient()}). + Complete() +} + +// NOTE: The 'path' attribute must follow a specific pattern and should not be modified directly here. +// Modifying the path for an invalid path can cause API server errors; failing to locate the webhook. +// +kubebuilder:webhook:path=/validate-serving-kserve-io-v1beta1-inferenceservice,mutating=false,failurePolicy=fail,sideEffects=None,groups=serving.kserve.io,resources=inferenceservices,verbs=create,versions=v1beta1,name=validating.isvc.odh-model-controller.opendatahub.io,admissionReviewVersions=v1 + +// InferenceServiceCustomValidator struct is responsible for validating the InferenceService resource +// when it is created, updated, or deleted. +// +// NOTE: The +kubebuilder:object:generate=false marker prevents controller-gen from generating DeepCopy methods, +// as this struct is used only for temporary operations and does not need to be deeply copied. +type InferenceServiceCustomValidator struct { + client client.Client +} + +var _ webhook.CustomValidator = &InferenceServiceCustomValidator{} + +// ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type InferenceService. +func (v *InferenceServiceCustomValidator) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + inferenceservice, ok := obj.(*servingv1beta1.InferenceService) + if !ok { + return nil, fmt.Errorf("expected a InferenceService object but got %T", obj) + } + logger := inferenceservicelog.WithValues("namespace", inferenceservice.Namespace, "isvc", inferenceservice.GetName()) + logger.Info("Validation for InferenceService upon creation") + + protectedNamespaces := make([]string, 3) + // hardcoding for now since there is no plan to install knative on other namespaces + protectedNamespaces[0] = "knative-serving" + _, meshNamespace := utils.GetIstioControlPlaneName(ctx, v.client) + protectedNamespaces[1] = meshNamespace + + appNamespace, err := utils.GetApplicationNamespace(ctx, v.client) + if err != nil { + return nil, err + } + protectedNamespaces[2] = appNamespace + + logger.Info("Filtering protected namespaces", "namespaces", protectedNamespaces) + for _, ns := range protectedNamespaces { + if inferenceservice.Namespace == ns { + logger.V(1).Info("Namespace is protected, the InferenceService will not be created") + return nil, errors.NewInvalid( + schema.GroupKind{Group: inferenceservice.GroupVersionKind().Group, Kind: inferenceservice.Kind}, + inferenceservice.GetName(), + field.ErrorList{ + field.Invalid(field.NewPath("metadata").Child("namespace"), inferenceservice.GetNamespace(), "specified namespace is protected"), + }) + } + } + + logger.Info("Namespace is not protected") + return nil, nil +} + +// ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type InferenceService. +func (v *InferenceServiceCustomValidator) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { + // Unused. Code below is from scaffolding + + inferenceservice, ok := newObj.(*servingv1beta1.InferenceService) + if !ok { + return nil, fmt.Errorf("expected a InferenceService object for the newObj but got %T", newObj) + } + inferenceservicelog.Info("Validation for InferenceService upon update", "name", inferenceservice.GetName()) + + // TODO(user): fill in your validation logic upon object update. + + return nil, nil +} + +// ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type InferenceService. +func (v *InferenceServiceCustomValidator) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + // Unused. Code below is from scaffolding + + inferenceservice, ok := obj.(*servingv1beta1.InferenceService) + if !ok { + return nil, fmt.Errorf("expected a InferenceService object but got %T", obj) + } + inferenceservicelog.Info("Validation for InferenceService upon deletion", "name", inferenceservice.GetName()) + + // TODO(user): fill in your validation logic upon object deletion. + + return nil, nil +} diff --git a/internal/webhook/serving/v1beta1/inferenceservice_webhook_test.go b/internal/webhook/serving/v1beta1/inferenceservice_webhook_test.go new file mode 100644 index 00000000..fb7b8b91 --- /dev/null +++ b/internal/webhook/serving/v1beta1/inferenceservice_webhook_test.go @@ -0,0 +1,93 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" + testutils "github.com/opendatahub-io/odh-model-controller/test/utils" +) + +var _ = Describe("InferenceService validator Webhook", func() { + var ( + validator InferenceServiceCustomValidator + meshNamespace, appsNamespace string + ) + + createInferenceService := func(namespace, name string) *kservev1beta1.InferenceService { + inferenceService := &kservev1beta1.InferenceService{} + err := testutils.ConvertToStructuredResource(KserveInferenceServicePath1, inferenceService) + Expect(err).NotTo(HaveOccurred()) + inferenceService.SetNamespace(namespace) + if len(name) != 0 { + inferenceService.Name = name + } + return inferenceService + } + + BeforeEach(func() { + _, meshNamespace = utils.GetIstioControlPlaneName(ctx, k8sClient) + appsNamespace, _ = utils.GetApplicationNamespace(ctx, k8sClient) + validator = InferenceServiceCustomValidator{client: k8sClient} + }) + + Context("When creating or updating InferenceService under Validating Webhook", func() { + It("Should allow the Inference Service in the test-model namespace", func() { + testNs := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-model", + Namespace: "test-model", + }, + } + Expect(k8sClient.Create(ctx, testNs)).Should(Succeed()) + + isvc := createInferenceService(testNs.Name, "test-isvc") + _, err := validator.ValidateCreate(ctx, isvc) + Expect(err).ShouldNot(HaveOccurred()) + }) + + It("Should not allow the Inference Service in the ServiceMesh namespace", func() { + isvc := createInferenceService(meshNamespace, "test-isvc") + _, err := validator.ValidateCreate(ctx, isvc) + Expect(err).To(HaveOccurred()) + }) + + It("Should not allow the Inference Service in the ApplicationsNamespace namespace", func() { + isvc := createInferenceService(appsNamespace, "test-isvc") + _, err := validator.ValidateCreate(ctx, isvc) + Expect(err).To(HaveOccurred()) + }) + + It("Should not allow the Inference Service in the knative-serving namespace", func() { + testNs := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "knative-serving", + Namespace: "knative-serving", + }, + } + Expect(k8sClient.Create(ctx, testNs)).Should(Succeed()) + isvc := createInferenceService(testNs.Name, "test-isvc") + _, err := validator.ValidateCreate(ctx, isvc) + Expect(err).To(HaveOccurred()) + }) + }) +}) diff --git a/internal/webhook/serving/v1beta1/testdata/kserve-openvino-inference-service-1.yaml b/internal/webhook/serving/v1beta1/testdata/kserve-openvino-inference-service-1.yaml new file mode 100644 index 00000000..62ee1087 --- /dev/null +++ b/internal/webhook/serving/v1beta1/testdata/kserve-openvino-inference-service-1.yaml @@ -0,0 +1,14 @@ +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: example-onnx-mnist + namespace: default +spec: + predictor: + model: + modelFormat: + name: onnx + runtime: kserve-ovms + storage: + key: testkey + path: /testpath/test \ No newline at end of file diff --git a/internal/webhook/serving/v1beta1/webhook_suite_test.go b/internal/webhook/serving/v1beta1/webhook_suite_test.go new file mode 100644 index 00000000..6e0f0435 --- /dev/null +++ b/internal/webhook/serving/v1beta1/webhook_suite_test.go @@ -0,0 +1,163 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "path/filepath" + "runtime" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + admissionv1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + apimachineryruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" + // +kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +const KserveInferenceServicePath1 = "./testdata/kserve-openvino-inference-service-1.yaml" + +var ( + cancel context.CancelFunc + cfg *rest.Config + ctx context.Context + k8sClient client.Client + testEnv *envtest.Environment +) + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Webhook Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{ + filepath.Join("..", "..", "..", "..", "config", "crd", "bases"), + filepath.Join("..", "..", "..", "..", "config", "crd", "external")}, + ErrorIfCRDPathMissing: true, + + // The BinaryAssetsDirectory is only required if you want to run the tests directly + // without call the makefile target test. If not informed it will look for the + // default path defined in controller-runtime which is /usr/local/kubebuilder/. + // Note that you must have the required binaries setup under the bin directory to perform + // the tests directly. When we run make test it will be setup and used automatically. + BinaryAssetsDirectory: filepath.Join("..", "..", "..", "..", "bin", "k8s", + fmt.Sprintf("1.31.0-%s-%s", runtime.GOOS, runtime.GOARCH)), + + WebhookInstallOptions: envtest.WebhookInstallOptions{ + Paths: []string{filepath.Join("..", "..", "..", "..", "config", "webhook")}, + }, + } + + var err error + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + scheme := apimachineryruntime.NewScheme() + utils.RegisterSchemes(scheme) + + err = admissionv1.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + + // +kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + // Create istio-system namespace + _, meshNamespace := utils.GetIstioControlPlaneName(ctx, k8sClient) + istioNamespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: meshNamespace, + Namespace: meshNamespace, + }, + } + Expect(k8sClient.Create(ctx, istioNamespace)).Should(Succeed()) + + // start webhook server using Manager. + webhookInstallOptions := &testEnv.WebhookInstallOptions + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme, + WebhookServer: webhook.NewServer(webhook.Options{ + Host: webhookInstallOptions.LocalServingHost, + Port: webhookInstallOptions.LocalServingPort, + CertDir: webhookInstallOptions.LocalServingCertDir, + }), + LeaderElection: false, + Metrics: metricsserver.Options{BindAddress: "0"}, + }) + Expect(err).NotTo(HaveOccurred()) + + err = SetupInferenceServiceWebhookWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + + // +kubebuilder:scaffold:webhook + + go func() { + defer GinkgoRecover() + err = mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred()) + }() + + // wait for the webhook server to get ready. + dialer := &net.Dialer{Timeout: time.Second} + addrPort := fmt.Sprintf("%s:%d", webhookInstallOptions.LocalServingHost, webhookInstallOptions.LocalServingPort) + Eventually(func() error { + conn, err := tls.DialWithDialer(dialer, "tcp", addrPort, &tls.Config{InsecureSkipVerify: true}) + if err != nil { + return err + } + + return conn.Close() + }).Should(Succeed()) +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + cancel() + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/main.go b/main.go deleted file mode 100644 index dd40a28b..00000000 --- a/main.go +++ /dev/null @@ -1,286 +0,0 @@ -/* -Copyright 2022. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "context" - "flag" - "os" - "slices" - "strconv" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/client-go/kubernetes" - knservingv1 "knative.dev/serving/pkg/apis/serving/v1" - "sigs.k8s.io/controller-runtime/pkg/builder" - "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/metrics/server" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - - // to ensure that exec-entrypoint and run can make use of them. - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) - _ "k8s.io/client-go/plugin/pkg/client/auth" - - "github.com/opendatahub-io/odh-model-controller/controllers" - "github.com/opendatahub-io/odh-model-controller/controllers/utils" - "github.com/opendatahub-io/odh-model-controller/controllers/webhook" - "istio.io/client-go/pkg/apis/security/v1beta1" - "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/healthz" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - - nimv1 "github.com/opendatahub-io/odh-model-controller/api/nim/v1" - - sigwebhook "sigs.k8s.io/controller-runtime/pkg/webhook" - //+kubebuilder:scaffold:imports -) - -var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") -) - -func init() { //nolint:gochecknoinits //reason this way we ensure schemes are always registered before we start anything - utils.RegisterSchemes(scheme) -} - -// ClusterRole permissions - -// +kubebuilder:rbac:groups=serving.kserve.io,resources=inferenceservices,verbs=get;list;watch;update -// +kubebuilder:rbac:groups=serving.kserve.io,resources=inferenceservices/finalizers,verbs=get;list;watch;update -// +kubebuilder:rbac:groups=serving.kserve.io,resources=servingruntimes,verbs=get;list;watch;create;update -// +kubebuilder:rbac:groups=serving.kserve.io,resources=servingruntimes/finalizers,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=networking.istio.io,resources=virtualservices,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=networking.istio.io,resources=virtualservices/finalizers,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=networking.istio.io,resources=gateways,verbs=get;list;watch;update;patch -// +kubebuilder:rbac:groups=security.istio.io,resources=peerauthentications,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=security.istio.io,resources=authorizationpolicies,verbs=get;list -// +kubebuilder:rbac:groups=telemetry.istio.io,resources=telemetries,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=maistra.io,resources=servicemeshmembers,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=maistra.io,resources=servicemeshmemberrolls,verbs=get;list;watch -// +kubebuilder:rbac:groups=maistra.io,resources=servicemeshcontrolplanes,verbs=get;list;watch;use -// +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=route.openshift.io,resources=routes/custom-host,verbs=create -// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterrolebindings;rolebindings,verbs=get;list;watch;create;update;patch;watch;delete -// +kubebuilder:rbac:groups=networking.k8s.io,resources=networkpolicies,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch -// +kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors;podmonitors,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=extensions,resources=ingresses,verbs=get;list;watch -// +kubebuilder:rbac:groups="",resources=namespaces;pods;endpoints,verbs=get;list;watch;create;update;patch -// +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;delete;patch -// +kubebuilder:rbac:groups="",resources=secrets;configmaps;serviceaccounts,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=authorino.kuadrant.io,resources=authconfigs,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=datasciencecluster.opendatahub.io,resources=datascienceclusters,verbs=get;list;watch -// +kubebuilder:rbac:groups=dscinitialization.opendatahub.io,resources=dscinitializations,verbs=get;list;watch -// +kubebuilder:rbac:groups=nim.opendatahub.io,resources=accounts,verbs=get;list;watch;update -// +kubebuilder:rbac:groups=nim.opendatahub.io,resources=accounts/status,verbs=get;list;watch;update -// +kubebuilder:rbac:groups=nim.opendatahub.io,resources=accounts/finalizers,verbs=update -// +kubebuilder:rbac:groups=template.openshift.io,resources=templates,verbs=get;list;watch;create;update;delete - -func getEnvAsBool(name string, defaultValue bool) bool { - valStr := os.Getenv(name) - if val, err := strconv.ParseBool(valStr); err == nil { - return val - } - return defaultValue -} - -func main() { - var metricsAddr string - var enableLeaderElection bool - var monitoringNS string - var probeAddr string - var enableMRInferenceServiceReconcile bool - - flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") - flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") - flag.BoolVar(&enableLeaderElection, "leader-elect", false, - "Enable leader election for controller manager. "+ - "Enabling this will ensure there is only one active controller manager.") - flag.StringVar(&monitoringNS, "monitoring-namespace", "", - "The Namespace where the monitoring stack's Prometheus resides.") - flag.BoolVar(&enableMRInferenceServiceReconcile, "model-registry-inference-reconcile", false, - "Enable model registry inference service reconciliation. ") - - opts := zap.Options{ - Development: true, - } - opts.BindFlags(flag.CommandLine) - flag.Parse() - - ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - - cfg := ctrl.GetConfigOrDie() - mgr, err := ctrl.NewManager(cfg, ctrl.Options{ - Scheme: scheme, - Metrics: server.Options{ - BindAddress: metricsAddr, - }, - HealthProbeBindAddress: probeAddr, - LeaderElection: enableLeaderElection, - LeaderElectionID: "odh-model-controller", - Client: client.Options{ - Cache: &client.CacheOptions{ - DisableFor: []client.Object{&v1beta1.AuthorizationPolicy{}}, - }, - }, - Cache: cache.Options{ - ByObject: map[client.Object]cache.ByObject{ - &corev1.Secret{}: { - Label: labels.SelectorFromSet(labels.Set{ - "opendatahub.io/managed": "true", - }), - }, - }, - }, - }) - - if err != nil { - setupLog.Error(err, "unable to start manager") - os.Exit(1) - } - - //Setup InferenceService controller - if err = (controllers.NewOpenshiftInferenceServiceReconciler( - mgr.GetClient(), - mgr.GetAPIReader(), - ctrl.Log.WithName("controllers").WithName("InferenceService"), - getEnvAsBool("MESH_DISABLED", false))). - SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "InferenceService") - os.Exit(1) - } - if err = (&controllers.StorageSecretReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("StorageSecret"), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "StorageSecret") - os.Exit(1) - } - - if err = (&controllers.KServeCustomCACertReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("KServeCustomeCABundleConfigMap"), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "KServeCustomeCABundleConfigMap") - os.Exit(1) - } - - if monitoringNS != "" { - setupLog.Info("Monitoring namespace provided, setting up monitoring controller.") - if err = (&controllers.MonitoringReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("MonitoringReconciler"), - MonitoringNS: monitoringNS, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "MonitoringReconciler") - os.Exit(1) - } - } - - if enableMRInferenceServiceReconcile { - setupLog.Info("Model registry inference service reconciliation enabled..") - if err = (controllers.NewModelRegistryInferenceServiceReconciler( - mgr.GetClient(), - ctrl.Log.WithName("controllers").WithName("ModelRegistryInferenceService"), - getEnvAsBool("MR_SKIP_TLS_VERIFY", false), - ctrl.GetConfigOrDie().BearerToken, - )).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "ModelRegistryInferenceServiceReconciler") - os.Exit(1) - } - } else { - setupLog.Info("Model registry inference service reconciliation disabled. To enable model registry " + - "reconciliation for InferenceService, please provide --model-registry-inference-reconcile flag.") - } - - kserveWithMeshEnabled, kserveWithMeshEnabledErr := utils.VerifyIfComponentIsEnabled(context.Background(), mgr.GetClient(), utils.KServeWithServiceMeshComponent) - if kserveWithMeshEnabledErr != nil { - setupLog.Error(kserveWithMeshEnabledErr, "could not determine if kserve have service mesh enabled") - } - - if kserveWithMeshEnabled { - ksvcValidatorWebhookSetupErr := builder.WebhookManagedBy(mgr). - For(&knservingv1.Service{}). - WithValidator(webhook.NewKsvcValidator(mgr.GetClient())). - Complete() - if ksvcValidatorWebhookSetupErr != nil { - setupLog.Error(err, "unable to setup Knative Service validating Webhook") - os.Exit(1) - } - - } else { - setupLog.Info("Skipping setup of Knative Service validating/mutating Webhook, because KServe Serverless setup seems to be disabled in the DataScienceCluster resource.") - } - - hookServer := mgr.GetWebhookServer() - isvcWebhook := &sigwebhook.Admission{ - Handler: &webhook.IsvcValidator{ - Client: mgr.GetClient(), - Decoder: admission.NewDecoder(mgr.GetScheme()), - }} - hookServer.Register("/validate-isvc-odh-service", isvcWebhook) - - kclient, kcErr := kubernetes.NewForConfig(cfg) - if kcErr != nil { - setupLog.Error(err, "unable to create clientset") - os.Exit(1) - } - - ctx := ctrl.SetupSignalHandler() - nimState := os.Getenv("NIM_STATE") - - if !slices.Contains([]string{"removed", ""}, nimState) { - if err = (&controllers.NimAccountReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("NimAccountReconciler"), - KClient: kclient, - }).SetupWithManager(mgr, ctx); err != nil { - setupLog.Error(err, "unable to create controller NIM Account controller") - os.Exit(1) - } - } - - if err = builder.WebhookManagedBy(mgr). - For(&nimv1.Account{}). - WithValidator(webhook.NewNimAccountValidator(mgr.GetClient())). - Complete(); err != nil { - setupLog.Error(err, "unable to setup NIM validating Webhook") - os.Exit(1) - } - - //+kubebuilder:scaffold:builder - - if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up health check") - os.Exit(1) - } - - if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up ready check") - os.Exit(1) - } - - setupLog.Info("starting manager") - if err := mgr.Start(ctx); err != nil { - setupLog.Error(err, "problem running manager") - os.Exit(1) - } -} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 1a291f0f..f0f5da5e 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package e2e import ( @@ -9,7 +25,7 @@ import ( kservev1alpha1 "github.com/kserve/kserve/pkg/apis/serving/v1alpha1" kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" routev1 "github.com/openshift/api/route/v1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" @@ -50,41 +66,120 @@ var ( kc *k8sclient.Clientset ) -// Run e2e tests using the Ginkgo runner. -// ODH model controller should be already running in the cluster where -// these tests will run against +var ( + // Optional Environment Variables: + // - PROMETHEUS_INSTALL_SKIP=true: Skips Prometheus Operator installation during test setup. + // - CERT_MANAGER_INSTALL_SKIP=true: Skips CertManager installation during test setup. + // These variables are useful if Prometheus or CertManager is already installed, avoiding + // re-installation and conflicts. + // skipPrometheusInstall = os.Getenv("PROMETHEUS_INSTALL_SKIP") == "true" + // skipCertManagerInstall = os.Getenv("CERT_MANAGER_INSTALL_SKIP") == "true" + // isPrometheusOperatorAlreadyInstalled will be set true when prometheus CRDs be found on the cluster + // isPrometheusOperatorAlreadyInstalled = false + // isCertManagerAlreadyInstalled will be set true when CertManager CRDs be found on the cluster + // isCertManagerAlreadyInstalled = false + + // projectImage is the name of the image which will be build and loaded + // with the code source changes to be tested. + projectImage = "example.com/odh-model-controller:v0.0.1" +) + +// TestE2E runs the end-to-end (e2e) test suite for the project. These tests execute in an isolated, +// temporary environment to validate project changes with the the purposed to be used in CI jobs. +// The default setup requires Kind, builds/loads the Manager Docker image locally, and installs +// CertManager and Prometheus. func TestE2E(t *testing.T) { RegisterFailHandler(Fail) - fmt.Fprintf(GinkgoWriter, "Starting ODH Model Controller suite\n") - RunSpecs(t, "ODH Model controller e2e suite") + _, _ = fmt.Fprintf(GinkgoWriter, "Starting odh-model-controller integration test suite\n") + RunSpecs(t, "e2e suite") } +// The following code is scaffolded code by kubebuilder. It is commented out, as MR +// provided custom code. We should try to restore it, if we ever have controller E2Es here. +// var _ = BeforeSuite(func() { +// By("Ensure that Prometheus is enabled") +// _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") +// +// By("generating files") +// cmd := exec.Command("make", "generate") +// _, err := utils.Run(cmd) +// ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to run make generate") +// +// By("generating manifests") +// cmd = exec.Command("make", "manifests") +// _, err = utils.Run(cmd) +// ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to run make manifests") +// +// By("building the manager(Operator) image") +// cmd = exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectImage)) +// _, err = utils.Run(cmd) +// ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the manager(Operator) image") +// +// // TODO(user): If you want to change the e2e test vendor from Kind, ensure the image is +// // built and available before running the tests. Also, remove the following block. +// By("loading the manager(Operator) image on Kind") +// err = utils.LoadImageToKindClusterWithName(projectImage) +// ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the manager(Operator) image into Kind") +// +// // The tests-e2e are intended to run on a temporary cluster that is created and destroyed for testing. +// // To prevent errors when tests run in environments with Prometheus or CertManager already installed, +// // we check for their presence before execution. +// // Setup Prometheus and CertManager before the suite if not skipped and if not already installed +// if !skipPrometheusInstall { +// By("checking if prometheus is installed already") +// isPrometheusOperatorAlreadyInstalled = utils.IsPrometheusCRDsInstalled() +// if !isPrometheusOperatorAlreadyInstalled { +// _, _ = fmt.Fprintf(GinkgoWriter, "Installing Prometheus Operator...\n") +// Expect(utils.InstallPrometheusOperator()).To(Succeed(), "Failed to install Prometheus Operator") +// } else { +// _, _ = fmt.Fprintf(GinkgoWriter, "WARNING: Prometheus Operator is already installed. Skipping installation...\n") +// } +// } +// if !skipCertManagerInstall { +// By("checking if cert manager is installed already") +// isCertManagerAlreadyInstalled = utils.IsCertManagerCRDsInstalled() +// if !isCertManagerAlreadyInstalled { +// _, _ = fmt.Fprintf(GinkgoWriter, "Installing CertManager...\n") +// Expect(utils.InstallCertManager()).To(Succeed(), "Failed to install CertManager") +// } else { +// _, _ = fmt.Fprintf(GinkgoWriter, "WARNING: CertManager is already installed. Skipping installation...\n") +// } +// } +// }) +// +// var _ = AfterSuite(func() { +// // Teardown Prometheus and CertManager after the suite if not skipped and if they were not already installed +// if !skipPrometheusInstall && !isPrometheusOperatorAlreadyInstalled { +// _, _ = fmt.Fprintf(GinkgoWriter, "Uninstalling Prometheus Operator...\n") +// utils.UninstallPrometheusOperator() +// } +// if !skipCertManagerInstall && !isCertManagerAlreadyInstalled { +// _, _ = fmt.Fprintf(GinkgoWriter, "Uninstalling CertManager...\n") +// utils.UninstallCertManager() +// } +// }) + var _ = BeforeSuite(func() { var err error ctx, cancel = context.WithCancel(context.TODO()) - // GetConfig(): If KUBECONFIG env variable is set, it is used to create // the client, else the inClusterConfig() is used. // Lastly if none of them are set, it uses $HOME/.kube/config to create the client. config, err := clientconfig.GetConfig() Expect(err).NotTo(HaveOccurred()) Expect(config).NotTo(BeNil()) - kc, err = k8sclient.NewForConfig(config) Expect(err).NotTo(HaveOccurred()) Expect(kc).NotTo(BeNil()) - // Custom client to manages resources cli, err = client.New(config, client.Options{Scheme: scheme}) Expect(err).NotTo(HaveOccurred()) Expect(cli).NotTo(BeNil()) - // Override kubectl cmd cmd, ok := os.LookupEnv(KubectlCmdEnv) if ok && cmd != "" { kubectl = cmd } - // Register API objects utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(kservev1alpha1.AddToScheme(scheme)) @@ -108,7 +203,6 @@ var _ = AfterEach(func() { // Expect(cli.DeleteAllOf(context.TODO(), &k8srbacv1.RoleBinding{}, inNamespace)).ToNot(HaveOccurred()) Expect(cli.DeleteAllOf(context.TODO(), &corev1.Secret{}, inNamespace)).ToNot(HaveOccurred()) }) - var _ = AfterSuite(func() { // TODO: remove all resources created in the current cluster }) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go new file mode 100644 index 00000000..f29bf4cb --- /dev/null +++ b/test/e2e/e2e_test.go @@ -0,0 +1,331 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/opendatahub-io/odh-model-controller/test/utils" +) + +// namespace where the project is deployed in +const namespace = "odh-model-controller-system" + +// serviceAccountName created for the project +const serviceAccountName = "odh-model-controller-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "odh-model-controller-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "odh-model-controller-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { + var controllerPodName string + + // Before running the tests, set up the environment by creating the namespace, + // installing CRDs, and deploying the controller. + BeforeAll(func() { + By("creating manager namespace") + cmd := exec.Command("kubectl", "create", "ns", namespace) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create namespace") + + By("installing CRDs") + cmd = exec.Command("make", "install") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs") + + By("deploying the controller-manager") + cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage)) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") + }) + + // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, + // and deleting the namespace. + AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + + By("undeploying the controller-manager") + cmd = exec.Command("make", "undeploy") + _, _ = utils.Run(cmd) + + By("uninstalling CRDs") + cmd = exec.Command("make", "uninstall") + _, _ = utils.Run(cmd) + + By("removing manager namespace") + cmd = exec.Command("kubectl", "delete", "ns", namespace) + _, _ = utils.Run(cmd) + }) + + // After each test, check for failures and collect logs, events, + // and pod descriptions for debugging. + AfterEach(func() { + specReport := CurrentSpecReport() + if specReport.Failed() { + By("Fetching controller manager pod logs") + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + controllerLogs, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Controller logs:\n %s", controllerLogs) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Controller logs: %s", err) + } + + By("Fetching Kubernetes events") + cmd = exec.Command("kubectl", "get", "events", "-n", namespace, "--sort-by=.lastTimestamp") + eventsOutput, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Kubernetes events:\n%s", eventsOutput) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Kubernetes events: %s", err) + } + + By("Fetching curl-metrics logs") + cmd = exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Metrics logs:\n %s", metricsOutput) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get curl-metrics logs: %s", err) + } + + By("Fetching controller manager pod description") + cmd = exec.Command("kubectl", "describe", "pod", controllerPodName, "-n", namespace) + podDescription, err := utils.Run(cmd) + if err == nil { + fmt.Println("Pod description:\n", podDescription) + } else { + fmt.Println("Failed to describe controller pod") + } + } + }) + + SetDefaultEventuallyTimeout(2 * time.Minute) + SetDefaultEventuallyPollingInterval(time.Second) + + Context("Manager", func() { + It("should run successfully", func() { + By("validating that the controller-manager pod is running as expected") + verifyControllerUp := func(g Gomega) { + // Get the name of the controller-manager pod + cmd := exec.Command("kubectl", "get", + "pods", "-l", "control-plane=controller-manager", + "-o", "go-template={{ range .items }}"+ + "{{ if not .metadata.deletionTimestamp }}"+ + "{{ .metadata.name }}"+ + "{{ \"\\n\" }}{{ end }}{{ end }}", + "-n", namespace, + ) + + podOutput, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") + podNames := utils.GetNonEmptyLines(podOutput) + g.Expect(podNames).To(HaveLen(1), "expected 1 controller pod running") + controllerPodName = podNames[0] + g.Expect(controllerPodName).To(ContainSubstring("controller-manager")) + + // Validate the pod's status + cmd = exec.Command("kubectl", "get", + "pods", controllerPodName, "-o", "jsonpath={.status.phase}", + "-n", namespace, + ) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Running"), "Incorrect controller-manager pod status") + } + Eventually(verifyControllerUp).Should(Succeed()) + }) + + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=odh-model-controller-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + Expect(err).NotTo(HaveOccurred()) + Expect(token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready") + } + Eventually(verifyMetricsEndpointReady).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + verifyMetricsServerStarted := func(g Gomega) { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"), + "Metrics server not yet started") + } + Eventually(verifyMetricsServerStarted).Should(Succeed()) + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Succeeded"), "curl pod in wrong status") + } + Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + Expect(metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + }) + + It("should provisioned cert-manager", func() { + By("validating that cert-manager has the certificate Secret") + verifyCertManager := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "secrets", "webhook-server-cert", "-n", namespace) + _, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + } + Eventually(verifyCertManager).Should(Succeed()) + }) + + It("should have CA injection for validating webhooks", func() { + By("checking CA injection for validating webhooks") + verifyCAInjection := func(g Gomega) { + cmd := exec.Command("kubectl", "get", + "validatingwebhookconfigurations.admissionregistration.k8s.io", + "odh-model-controller-validating-webhook-configuration", + "-o", "go-template={{ range .webhooks }}{{ .clientConfig.caBundle }}{{ end }}") + vwhOutput, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(vwhOutput)).To(BeNumerically(">", 10)) + } + Eventually(verifyCAInjection).Should(Succeed()) + }) + + // +kubebuilder:scaffold:e2e-webhooks-checks + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // Expect(metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) + }) +}) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + verifyTokenCreation := func(g Gomega) { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred()) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal(output, &token) + g.Expect(err).NotTo(HaveOccurred()) + + out = token.Status.Token + } + Eventually(verifyTokenCreation).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + Expect(metricsOutput).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutput +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/test/e2e/modelregistry_controller_e2e_test.go b/test/e2e/modelregistry_controller_e2e_test.go index f98d6c57..02d6b17c 100644 --- a/test/e2e/modelregistry_controller_e2e_test.go +++ b/test/e2e/modelregistry_controller_e2e_test.go @@ -10,11 +10,12 @@ import ( "github.com/kubeflow/model-registry/pkg/openapi" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/opendatahub-io/odh-model-controller/controllers/constants" - "github.com/opendatahub-io/odh-model-controller/test/utils" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/opendatahub-io/odh-model-controller/internal/controller/constants" + "github.com/opendatahub-io/odh-model-controller/test/utils" ) var ( @@ -82,7 +83,8 @@ var _ = Describe("ModelRegistry controller e2e", func() { AfterEach(func() { By("removing finalizers from inference service") - _, err := utils.Run(exec.Command(kubectl, "patch", "inferenceservice", "dummy-inference-service", "--type", "json", "--patch", "[ { \"op\": \"remove\", \"path\": \"/metadata/finalizers\" } ]")) + _, err := utils.Run(exec.Command(kubectl, "patch", "inferenceservice", "dummy-inference-service", + "--type", "json", "--patch", "[ { \"op\": \"remove\", \"path\": \"/metadata/finalizers\" } ]")) Expect(err).ToNot(HaveOccurred()) }) @@ -229,17 +231,19 @@ var _ = Describe("ModelRegistry controller e2e", func() { // simulate ServingEnvironment creation envName := WorkingNamespace - _, _, err := modelRegistryClient.ModelRegistryServiceAPI.CreateServingEnvironment(ctx).ServingEnvironmentCreate(openapi.ServingEnvironmentCreate{ - Name: &envName, - }).Execute() + _, _, err := modelRegistryClient.ModelRegistryServiceAPI.CreateServingEnvironment(ctx). + ServingEnvironmentCreate(openapi.ServingEnvironmentCreate{ + Name: &envName, + }).Execute() Expect(err).ToNot(HaveOccurred()) - inferenceService, _, err = modelRegistryClient.ModelRegistryServiceAPI.CreateInferenceService(ctx).InferenceServiceCreate(openapi.InferenceServiceCreate{ - Name: &versionName, - DesiredState: openapi.INFERENCESERVICESTATE_DEPLOYED.Ptr(), - RegisteredModelId: *registeredModel.Id, - ServingEnvironmentId: "3", - }).Execute() + inferenceService, _, err = modelRegistryClient.ModelRegistryServiceAPI.CreateInferenceService(ctx). + InferenceServiceCreate(openapi.InferenceServiceCreate{ + Name: &versionName, + DesiredState: openapi.INFERENCESERVICESTATE_DEPLOYED.Ptr(), + RegisteredModelId: *registeredModel.Id, + ServingEnvironmentId: "3", + }).Execute() Expect(err).ToNot(HaveOccurred()) Expect(inferenceService.GetId()).To(Equal("4")) @@ -296,7 +300,7 @@ func deployAndCheckModelRegistry() *openapi.APIClient { mrServiceList := &corev1.ServiceList{} err = cli.List(ctx, mrServiceList, opts...) Expect(err).ToNot(HaveOccurred()) - Expect(len(mrServiceList.Items)).To(Equal(1)) + Expect(mrServiceList.Items).To(HaveLen(1)) var restApiPort *int32 for _, port := range mrServiceList.Items[0].Spec.Ports { @@ -382,31 +386,36 @@ func fillModelRegistryContent(mr *openapi.APIClient) { registeredModel, _, err = mr.ModelRegistryServiceAPI.FindRegisteredModel(ctx).Name(modelName).Execute() if err != nil { // register a new model - registeredModel, _, err = mr.ModelRegistryServiceAPI.CreateRegisteredModel(ctx).RegisteredModelCreate(openapi.RegisteredModelCreate{ - Name: modelName, - }).Execute() + registeredModel, _, err = mr.ModelRegistryServiceAPI.CreateRegisteredModel(ctx).RegisteredModelCreate( + openapi.RegisteredModelCreate{ + Name: modelName, + }).Execute() Expect(err).ToNot(HaveOccurred()) } - modelVersion, _, err = mr.ModelRegistryServiceAPI.FindModelVersion(ctx).Name(versionName).ParentResourceId(*registeredModel.Id).Execute() + modelVersion, _, err = mr.ModelRegistryServiceAPI.FindModelVersion(ctx).Name(versionName). + ParentResourceId(*registeredModel.Id).Execute() if err != nil { - modelVersion, _, err = mr.ModelRegistryServiceAPI.CreateModelVersion(ctx).ModelVersionCreate(openapi.ModelVersionCreate{ - Name: versionName, - RegisteredModelId: *registeredModel.Id, - }).Execute() + modelVersion, _, err = mr.ModelRegistryServiceAPI.CreateModelVersion(ctx).ModelVersionCreate( + openapi.ModelVersionCreate{ + Name: versionName, + RegisteredModelId: *registeredModel.Id, + }).Execute() Expect(err).ToNot(HaveOccurred()) } modelArtifactName := fmt.Sprintf("%s-artifact", versionName) - modelArtifact, _, err = mr.ModelRegistryServiceAPI.FindModelArtifact(ctx).Name(modelArtifactName).ParentResourceId(*modelVersion.Id).Execute() + modelArtifact, _, err = mr.ModelRegistryServiceAPI.FindModelArtifact(ctx).Name(modelArtifactName). + ParentResourceId(*modelVersion.Id).Execute() if err != nil { - modelArtifact, _, err = mr.ModelRegistryServiceAPI.CreateModelArtifact(ctx).ModelArtifactCreate(openapi.ModelArtifactCreate{ - Name: &modelArtifactName, - ModelFormatName: &modelFormatName, - ModelFormatVersion: &modelFormatVersion, - StorageKey: &storageKey, - StoragePath: &storagePath, - }).Execute() + modelArtifact, _, err = mr.ModelRegistryServiceAPI.CreateModelArtifact(ctx).ModelArtifactCreate( + openapi.ModelArtifactCreate{ + Name: &modelArtifactName, + ModelFormatName: &modelFormatName, + ModelFormatVersion: &modelFormatVersion, + StorageKey: &storageKey, + StoragePath: &storagePath, + }).Execute() Expect(err).ToNot(HaveOccurred()) } } diff --git a/test/utils/NamespaceHolder.go b/test/utils/NamespaceHolder.go new file mode 100644 index 00000000..95a1bc20 --- /dev/null +++ b/test/utils/NamespaceHolder.go @@ -0,0 +1,58 @@ +package utils + +import ( + "context" + "math/rand" + + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var Namespaces NamespaceHolder + +type NamespaceHolder struct { + namespaces []string +} + +func init() { + Namespaces = NamespaceHolder{} +} + +func createTestNamespaceName() string { + n := 5 + letterRunes := []rune("abcdefghijklmnopqrstuvwxyz") + + b := make([]rune, n) + for i := range b { + b[i] = letterRunes[rand.Intn(len(letterRunes))] + } + return "test-ns-" + string(b) +} + +func (n *NamespaceHolder) Get() string { + ns := createTestNamespaceName() + n.namespaces = append(n.namespaces, ns) + return ns +} + +func (n *NamespaceHolder) All() []string { + return n.namespaces +} + +func (n *NamespaceHolder) Clear() { + n.namespaces = []string{} +} + +func (n *NamespaceHolder) Create(ctx context.Context, cli client.Client) *corev1.Namespace { + testNs := Namespaces.Get() + testNamespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNs, + Namespace: testNs, + }, + } + Expect(cli.Create(ctx, testNamespace)).Should(Succeed()) + return testNamespace +} diff --git a/test/utils/utils.go b/test/utils/utils.go index a5f19111..155c58d4 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -1,37 +1,204 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package utils import ( + "bufio" + "bytes" + "context" "fmt" "os" "os/exec" + "reflect" "strings" + "time" + + . "github.com/onsi/ginkgo/v2" //nolint:golint,revive + corev1 "k8s.io/api/core/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" - //nolint:golint,revive - . "github.com/onsi/ginkgo" - "gopkg.in/yaml.v2" + "github.com/opendatahub-io/odh-model-controller/internal/controller/utils" ) +const ( + prometheusOperatorVersion = "v0.77.1" + prometheusOperatorURL = "https://github.com/prometheus-operator/prometheus-operator/" + + "releases/download/%s/bundle.yaml" + + certmanagerVersion = "v1.16.0" + certmanagerURLTmpl = "https://github.com/jetstack/cert-manager/releases/download/%s/cert-manager.yaml" +) + +func warnError(err error) { + _, _ = fmt.Fprintf(GinkgoWriter, "warning: %v\n", err) +} + // Run executes the provided command within this context -func Run(cmd *exec.Cmd) ([]byte, error) { +func Run(cmd *exec.Cmd) (string, error) { dir, _ := GetProjectDir() cmd.Dir = dir - fmt.Fprintf(GinkgoWriter, "running dir: %s\n", cmd.Dir) - // To allow make commands be executed from the project directory which is subdir on SDK repo - // TODO:(user) You might not need the following code if err := os.Chdir(cmd.Dir); err != nil { - fmt.Fprintf(GinkgoWriter, "chdir dir: %s\n", err) + _, _ = fmt.Fprintf(GinkgoWriter, "chdir dir: %s\n", err) } cmd.Env = append(os.Environ(), "GO111MODULE=on") command := strings.Join(cmd.Args, " ") - fmt.Fprintf(GinkgoWriter, "running: %s\n", command) + _, _ = fmt.Fprintf(GinkgoWriter, "running: %s\n", command) output, err := cmd.CombinedOutput() if err != nil { - return output, fmt.Errorf("%s failed with error: (%v) %s", command, err, string(output)) + return string(output), fmt.Errorf("%s failed with error: (%v) %s", command, err, string(output)) + } + + return string(output), nil +} + +// InstallPrometheusOperator installs the prometheus Operator to be used to export the enabled metrics. +func InstallPrometheusOperator() error { + url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) + cmd := exec.Command("kubectl", "create", "-f", url) + _, err := Run(cmd) + return err +} + +// UninstallPrometheusOperator uninstalls the prometheus +func UninstallPrometheusOperator() { + url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) + cmd := exec.Command("kubectl", "delete", "-f", url) + if _, err := Run(cmd); err != nil { + warnError(err) + } +} + +// IsPrometheusCRDsInstalled checks if any Prometheus CRDs are installed +// by verifying the existence of key CRDs related to Prometheus. +func IsPrometheusCRDsInstalled() bool { + // List of common Prometheus CRDs + prometheusCRDs := []string{ + "prometheuses.monitoring.coreos.com", + "prometheusrules.monitoring.coreos.com", + "prometheusagents.monitoring.coreos.com", + } + + cmd := exec.Command("kubectl", "get", "crds", "-o", "custom-columns=NAME:.metadata.name") + output, err := Run(cmd) + if err != nil { + return false + } + crdList := GetNonEmptyLines(output) + for _, crd := range prometheusCRDs { + for _, line := range crdList { + if strings.Contains(line, crd) { + return true + } + } + } + + return false +} + +// UninstallCertManager uninstalls the cert manager +func UninstallCertManager() { + url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) + cmd := exec.Command("kubectl", "delete", "-f", url) + if _, err := Run(cmd); err != nil { + warnError(err) + } +} + +// InstallCertManager installs the cert manager bundle. +func InstallCertManager() error { + url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) + cmd := exec.Command("kubectl", "apply", "-f", url) + if _, err := Run(cmd); err != nil { + return err + } + // Wait for cert-manager-webhook to be ready, which can take time if cert-manager + // was re-installed after uninstalling on a cluster. + cmd = exec.Command("kubectl", "wait", "deployment.apps/cert-manager-webhook", + "--for", "condition=Available", + "--namespace", "cert-manager", + "--timeout", "5m", + ) + + _, err := Run(cmd) + return err +} + +// IsCertManagerCRDsInstalled checks if any Cert Manager CRDs are installed +// by verifying the existence of key CRDs related to Cert Manager. +func IsCertManagerCRDsInstalled() bool { + // List of common Cert Manager CRDs + certManagerCRDs := []string{ + "certificates.cert-manager.io", + "issuers.cert-manager.io", + "clusterissuers.cert-manager.io", + "certificaterequests.cert-manager.io", + "orders.acme.cert-manager.io", + "challenges.acme.cert-manager.io", + } + + // Execute the kubectl command to get all CRDs + cmd := exec.Command("kubectl", "get", "crds") + output, err := Run(cmd) + if err != nil { + return false } - return output, nil + // Check if any of the Cert Manager CRDs are present + crdList := GetNonEmptyLines(output) + for _, crd := range certManagerCRDs { + for _, line := range crdList { + if strings.Contains(line, crd) { + return true + } + } + } + + return false +} + +// LoadImageToKindClusterWithName loads a local docker image to the kind cluster +func LoadImageToKindClusterWithName(name string) error { + cluster := "kind" + if v, ok := os.LookupEnv("KIND_CLUSTER"); ok { + cluster = v + } + kindOptions := []string{"load", "docker-image", name, "--name", cluster} + cmd := exec.Command("kind", kindOptions...) + _, err := Run(cmd) + return err +} + +// GetNonEmptyLines converts given command output string into individual objects +// according to line breakers, and ignores the empty elements in it. +func GetNonEmptyLines(output string) []string { + var res []string + elements := strings.Split(output, "\n") + for _, element := range elements { + if element != "" { + res = append(res, element) + } + } + + return res } // GetProjectDir will return the directory where the project is @@ -44,43 +211,98 @@ func GetProjectDir() (string, error) { return wd, nil } -func ConvertToStructuredResource(path string, out interface{}) (err error) { - data, err := os.ReadFile(path) +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) if err != nil { return err } - // Unmarshal the YAML data into the struct - err = yaml.Unmarshal(data, out) + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) if err != nil { return err } - return nil + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) } -// InstallModelControllerOperator build and deploy the odh model controller -// Assume the docker image has been already built -func InstallModelControllerOperator() (err error) { - cmd := exec.Command("make", "install") - _, err = Run(cmd) +func ConvertToStructuredResource(path string, out runtime.Object) error { + data, err := os.ReadFile(path) if err != nil { - return + return err } - cmd = exec.Command("make", "deploy") - _, err = Run(cmd) - return + return utils.ConvertToStructuredResource(data, out) } -// UninstallModelControllerOperator undeploy the odh model controller -func UninstallModelControllerOperator() (err error) { - cmd := exec.Command("make", "undeploy") - _, err = Run(cmd) +func ConvertToUnstructuredResource(path string, out *unstructured.Unstructured) error { + data, err := os.ReadFile(path) if err != nil { - return + return err } - cmd = exec.Command("make", "uninstall") - _, err = Run(cmd) - return + return utils.ConvertToUnstructuredResource(data, out) +} + +// CompareConfigMap checks if two ConfigMap data are equal, if not return false +func CompareConfigMap(s1 *corev1.ConfigMap, s2 *corev1.ConfigMap) bool { + // Two ConfigMap will be equal if the data is identical + return reflect.DeepEqual(s1.Data, s2.Data) +} + +func WaitForConfigMap(cli client.Client, namespace, configMapName string, maxTries int, + delay time.Duration) (*corev1.ConfigMap, error) { + time.Sleep(delay) + + ctx := context.Background() + configMap := &corev1.ConfigMap{} + for try := 1; try <= maxTries; try++ { + err := cli.Get(ctx, client.ObjectKey{Namespace: namespace, Name: configMapName}, configMap) + if err == nil { + return configMap, nil + } + if !apierrs.IsNotFound(err) { + return nil, fmt.Errorf("failed to get configmap %s/%s: %v", namespace, configMapName, err) + } + + if try > maxTries { + time.Sleep(1 * time.Second) + return nil, err + } + } + return configMap, nil }