diff --git a/infra/feast-operator/demo/README.md b/infra/feast-operator/demo/README.md new file mode 100644 index 0000000000..77eaa599f3 --- /dev/null +++ b/infra/feast-operator/demo/README.md @@ -0,0 +1,97 @@ +# Installing the sample notebook +Follow this procedure to install and configure a sample Jupyter Notebook in your Kubernetes environment that can +connect a deployed `FeatureStore` instance. + +## Requirement +* `FeatureStore` instance deployed using the Feast Operator. +* Local environment logged to the target Kubernetes cluster with `admin` privileges. +* `kubectl` CLI +* A Notebook instance created in the Kubernetes cluster (either with KubeFlow, OpedDataHub or RedHat OpenShift AI) + * No specific image is needed for the purpose, assuming that it includes the Python interpreter. + +## Mounting the Feast resources +**Important Notes**: +* This procedure causes a restart of the running notebook to apply the mounted resources. Save your changes, before +proceeding. +* This procedure updates the `Notebook` custom resource and must be executed just once. Next executions will break +the consistency of the manifest. In this case, edit the resource manifest and remove the duplicated settings. + +Launch this command to mount the Feast client ConfigMap and the required TLS secrets to the only Notebook in the namespace: +```bash +./init_feast_notebook.sh +``` + +In case you have multiple Notebooks or Feast instances, provide their names using the optional parameters: +```bash +% ./init_feast_notebook.sh --help +Usage: init_feast_notebook.sh [--notebook ] [--feast ] [--help] + +Options: + --notebook Specify the notebook name. Defaults to the only Notebook instance in the current namespace. + --feast Specify the Feast instance name. Defaults to the only Feast instance in the current namespace. + --help Display this help message. +``` + +Output example: +```console +Notebook Name: feast-client +Feast Name: example +ConfigMap Mount Path: /feast/feature_repository +Connecting Notebook 'feast-client' with Feast 'example'... +Patching Notebook feast-client with ConfigMap feast-example-client +notebook.kubeflow.org/feast-client patched +Offline TLS Mount Path: +Online TLS Mount Path: /tls/online +Registry TLS Mount Path: /tls/registry +Patching Notebook feast-client with Secret feast-example-online-tls mounted at /tls/online +notebook.kubeflow.org/feast-client patched +Patching Notebook feast-client with Secret feast-example-registry-tls mounted at /tls/registry +notebook.kubeflow.org/feast-client patched +``` + +## Validating the Notebook + +Wait until the notebook is ready (replace NOTEBOOK_NAME with your notebook name): +```bash +kubectl wait --for=condition=ready pod/NOTEBOOK_NAME-0 --timeout=2m +``` + +Then verify the content of the mounted folders (update `/feast` and `/tls` with your actual paths, if needed) +```bash +kubectl exec pod/NOTEBOOK_NAME-0 -c NOTEBOOK_NAME -- bash -c "find /feast; find /tls" +``` + +Output example: +```console +/feast +/feast/feature_repository +/feast/feature_repository/..2024_12_05_09_26_57.3976272882 +/feast/feature_repository/..2024_12_05_09_26_57.3976272882/feature_store.yaml +/feast/feature_repository/..data +/feast/feature_repository/feature_store.yaml +/tls +/tls/online +/tls/online/tls.key +/tls/online/tls.crt +/tls/online/..data +/tls/online/..2024_12_05_09_26_57.1747038009 +/tls/online/..2024_12_05_09_26_57.1747038009/tls.key +/tls/online/..2024_12_05_09_26_57.1747038009/tls.crt +/tls/registry +/tls/registry/tls.key +/tls/registry/tls.crt +/tls/registry/..data +/tls/registry/..2024_12_05_09_26_57.3730005170 +/tls/registry/..2024_12_05_09_26_57.3730005170/tls.key +/tls/registry/..2024_12_05_09_26_57.3730005170/tls.crt +``` + +## Importing the quickstart Jupyter Notebook +The provided [quickstart](./quickstart.ipynb) notebook provides you a quick-start client application to connect the +deployed Feast instance: +* Install `feast` from either `git` (for latest developments) or `pypi` (for released versions). +* Connect to the deplpoyed Feast instance and validate the content: + * Using `feast` CLI. + * With `feast` python package. + +Copy the notebook in your workbench, using either `git clone`, `kubectl cp` or `wget`/`curl` and enjoy the experience! diff --git a/infra/feast-operator/demo/init_feast_notebook.sh b/infra/feast-operator/demo/init_feast_notebook.sh new file mode 100755 index 0000000000..80f71f55ab --- /dev/null +++ b/infra/feast-operator/demo/init_feast_notebook.sh @@ -0,0 +1,161 @@ +#!/bin/bash + +NOTEBOOK_NAME="" +FEAST_NAME="" +CM_MOUNT_PATH="/feast/feature_repository" + +show_help() { + echo "Usage: init_feast_notebook.sh [--notebook ] [--feast ] [--help]" + echo + echo "Options:" + echo " --notebook Specify the notebook name. Defaults to the only Notebook instance in the current namespace." + echo " --feast Specify the Feast instance name. Defaults to the only Feast instance in the current namespace." + echo " --help Display this help message." + exit 0 +} + +get_single_resource_name() { + local resource_type=$1 + local resource_count + resource_count=$(kubectl get "$resource_type" --no-headers | wc -l) + + if [ "$resource_count" -eq 1 ]; then + kubectl get "$resource_type" --no-headers | awk '{print $1}' + else + echo "" + fi +} + +patch_config_map() { + local notebook_name=$1 + local config_map_name=$2 + local mount_path=$3 + echo "Patching Notebook $notebook_name with ConfigMap $config_map_name" + kubectl patch notebook "$notebook_name" -n feast --type='json' -p="[ + { + \"op\": \"add\", + \"path\": \"/spec/template/spec/volumes/-\", + \"value\": { + \"name\": \"$config_map_name\", + \"configMap\": { + \"name\": \"$config_map_name\" + } + } + }, + { + \"op\": \"add\", + \"path\": \"/spec/template/spec/containers/0/volumeMounts/-\", + \"value\": { + \"name\": \"$config_map_name\", + \"mountPath\": \"$mount_path\" + } + } + ]" +} + +get_tls_secret_name() { + local service_type=$1 + local config_map_name=$2 + kubectl get cm $config_map_name -oyaml | yq '.data."feature_store.yaml"' | yq ".${service_type}.cert" | xargs dirname +} + +patch_tls_secret(){ + local notebook_name=$1 + local secret_name=$2 + local mount_path=$3 + + echo "Patching Notebook $notebook_name with Secret $secret_name mounted at $mount_path" + kubectl patch notebook "$notebook_name" -n feast --type='json' -p="[ + { + \"op\": \"add\", + \"path\": \"/spec/template/spec/volumes/-\", + \"value\": { + \"name\": \"$secret_name\", + \"secret\": { + \"defaultMode\": 420, + \"secretName\": \"$secret_name\" + } + } + }, + { + \"op\": \"add\", + \"path\": \"/spec/template/spec/containers/0/volumeMounts/-\", + \"value\": { + \"name\": \"$secret_name\", + \"mountPath\": \"$mount_path\" + } + } + ]" +} + +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + --notebook) + NOTEBOOK_NAME="$2" + shift + shift + ;; + --feast) + FEAST_NAME="$2" + shift + shift + ;; + --help) + show_help + ;; + *) + echo "Unknown option $1" + exit 1 + ;; + esac +done + +if [ -z "$NOTEBOOK_NAME" ]; then + NOTEBOOK_NAME=$(get_single_resource_name "notebook") + if [ -z "$NOTEBOOK_NAME" ]; then + echo "Error: Multiple or no Notebook instances found. Specify the --notebook parameter." + exit 1 + fi +fi + +if [ -z "$FEAST_NAME" ]; then + FEAST_NAME=$(get_single_resource_name "feast") + if [ -z "$FEAST_NAME" ]; then + echo "Error: Multiple or no Feast instances found. Specify the --feast parameter." + exit 1 + fi +fi + +echo "Notebook Name: $NOTEBOOK_NAME" +echo "Feast Name: $FEAST_NAME" +echo "ConfigMap Mount Path: $CM_MOUNT_PATH" + +echo "Connecting Notebook '$NOTEBOOK_NAME' with Feast '$FEAST_NAME'..." + +client_configmap_name="feast-${FEAST_NAME}-client" +patch_config_map $NOTEBOOK_NAME $client_configmap_name $CM_MOUNT_PATH + +offline_tls_mount_path=$(get_tls_secret_name "offline_store" $client_configmap_name) +online_tls_mount_path=$(get_tls_secret_name "online_store" $client_configmap_name) +registry_tls_mount_path=$(get_tls_secret_name "registry" $client_configmap_name) +echo "Offline TLS Mount Path: $offline_tls_secret_name" +echo "Online TLS Mount Path: $online_tls_mount_path" +echo "Registry TLS Mount Path: $registry_tls_mount_path" + +if [ "$offline_tls_mount_path" != "." ]; then + secret_name="feast-${FEAST_NAME}-offline-tls" + patch_tls_secret $NOTEBOOK_NAME $secret_name ${offline_tls_mount_path} +fi +if [ "$online_tls_mount_path" != "." ]; then + secret_name="feast-${FEAST_NAME}-online-tls" + patch_tls_secret $NOTEBOOK_NAME $secret_name ${online_tls_mount_path} +fi +if [ "$registry_tls_mount_path" != "." ]; then + secret_name="feast-${FEAST_NAME}-registry-tls" + patch_tls_secret $NOTEBOOK_NAME $secret_name ${registry_tls_mount_path} +fi + + + + diff --git a/infra/feast-operator/demo/quickstart.ipynb b/infra/feast-operator/demo/quickstart.ipynb new file mode 100644 index 0000000000..f0b7cb97c4 --- /dev/null +++ b/infra/feast-operator/demo/quickstart.ipynb @@ -0,0 +1,355 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "45d032f1", + "metadata": {}, + "source": [ + "# Feast Notebook Quickstart" + ] + }, + { + "cell_type": "markdown", + "id": "66da1352-df53-414c-a83a-ba971e34f677", + "metadata": {}, + "source": [ + "## Install feast package" + ] + }, + { + "cell_type": "markdown", + "id": "97dbe224-5606-4aa9-b01b-ea3d8657d828", + "metadata": { + "tags": [] + }, + "source": [ + "### From git\n", + "Consider this option if you need to use features under development not yet released." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "fc2d77c7-f15d-4782-a84b-bd7b71d8e1cf", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cloning into 'feast'...\n", + "remote: Enumerating objects: 66378, done.\u001b[K\n", + "remote: Counting objects: 100% (1139/1139), done.\u001b[K\n", + "remote: Compressing objects: 100% (622/622), done.\u001b[K\n", + "remote: Total 66378 (delta 628), reused 828 (delta 435), pack-reused 65239 (from 1)\u001b[K\n", + "Receiving objects: 100% (66378/66378), 88.10 MiB | 29.56 MiB/s, done.\n", + "Resolving deltas: 100% (41982/41982), done.\n" + ] + } + ], + "source": [ + "!git clone https://github.com/feast-dev/feast.git" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "95ac710f-90cf-4de9-b80f-e55655cb3acc", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.2.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install -q -e \"feast[dev,postgres]\"" + ] + }, + { + "cell_type": "markdown", + "id": "2a236cf5-7e23-4001-9b0b-f0fe167aceeb", + "metadata": {}, + "source": [ + "**Note**: Additional dependencies might be needed according to the actual feature store components. Recommendation is to install them using the extras dependencies\n", + "defined in `setup.py` to guarantee consistency against the validated versions." + ] + }, + { + "cell_type": "markdown", + "id": "2a2ab2cf-e326-43ef-85b2-708d4faf8e45", + "metadata": {}, + "source": [ + "### From python package repository\n", + "Consider this option if you need to use features already released." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "8d8b53e1-cfd0-4956-aff1-18d09944818f", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Use this if you want a stable version\n", + "# %pip install -q \"feast[postgres]\"" + ] + }, + { + "cell_type": "markdown", + "id": "0b558cf9-4625-4fb8-b037-3747e7b34ffc", + "metadata": {}, + "source": [ + "## Validate the Notebook file system\n", + "Verify that the client ConfigMap and the TLS secrets have been properly mounted." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "d1619ccd-420c-4f6c-b397-b383c2ad4910", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/feast\n", + "/feast/feature_repository\n", + "/feast/feature_repository/..2024_12_05_10_02_40.4029380524\n", + "/feast/feature_repository/..2024_12_05_10_02_40.4029380524/feature_store.yaml\n", + "/feast/feature_repository/..data\n", + "/feast/feature_repository/feature_store.yaml\n" + ] + } + ], + "source": [ + "!find /feast" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "b50b9867-9409-4991-9e53-f012e28eabbd", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "project: sample\n", + "provider: local\n", + "offline_store:\n", + " host: feast-example-offline.feast.svc.cluster.local\n", + " type: remote\n", + " port: 80\n", + "online_store:\n", + " path: https://feast-example-online.feast.svc.cluster.local:443\n", + " type: remote\n", + " cert: /tls/online/tls.crt\n", + "registry:\n", + " path: feast-example-registry.feast.svc.cluster.local:443\n", + " registry_type: remote\n", + " cert: /tls/registry/tls.crt\n", + "auth:\n", + " type: no_auth\n", + "entity_key_serialization_version: 3\n" + ] + } + ], + "source": [ + "!cat /feast/feature_repository/feature_store.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "4deea17c-1379-4cd9-a44c-dd2a98a712c6", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/tls\n", + "/tls/online\n", + "/tls/online/tls.key\n", + "/tls/online/tls.crt\n", + "/tls/online/..data\n", + "/tls/online/..2024_12_05_10_02_40.3439156357\n", + "/tls/online/..2024_12_05_10_02_40.3439156357/tls.key\n", + "/tls/online/..2024_12_05_10_02_40.3439156357/tls.crt\n", + "/tls/registry\n", + "/tls/registry/tls.key\n", + "/tls/registry/tls.crt\n", + "/tls/registry/..data\n", + "/tls/registry/..2024_12_05_10_02_40.1732177020\n", + "/tls/registry/..2024_12_05_10_02_40.1732177020/tls.key\n", + "/tls/registry/..2024_12_05_10_02_40.1732177020/tls.crt\n" + ] + } + ], + "source": [ + "!find /tls" + ] + }, + { + "cell_type": "markdown", + "id": "cda33cb0-1f36-4265-b164-287780007832", + "metadata": {}, + "source": [ + "## Validate communication using feast CLI\n", + "Run basic commands to verify the content of the feature store." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "585bded7-b833-4479-80ef-a95a7c08e9af", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "env: FEATURE_REPO=/feast/feature_repository\n" + ] + } + ], + "source": [ + "%env FEATURE_REPO=/feast/feature_repository" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "28dad540-9557-420b-aafb-df1dd132ddb0", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME DESCRIPTION TYPE\n", + "driver ValueType.UNKNOWN\n" + ] + } + ], + "source": [ + "!feast -c $FEATURE_REPO entities list " + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "fe61db0b-21f6-4e50-a713-183a99984981", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME ENTITIES TYPE\n", + "driver_hourly_stats_fresh {'driver'} FeatureView\n", + "driver_hourly_stats {'driver'} FeatureView\n", + "transformed_conv_rate_fresh {'driver'} OnDemandFeatureView\n", + "transformed_conv_rate {'driver'} OnDemandFeatureView\n" + ] + } + ], + "source": [ + "!feast -c $FEATURE_REPO feature-views list" + ] + }, + { + "cell_type": "markdown", + "id": "7c5beefe-ef36-42b9-bdf8-4e974b15c260", + "metadata": {}, + "source": [ + "## Validate communication using feast modules\n", + "Use feast python modules to verify and manage the feature store." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "6b2da91d-f206-4f15-bc39-2c12234e2511", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "vals_to_add\n", + "driver_stats_push_source\n", + "driver_hourly_stats_source\n" + ] + } + ], + "source": [ + "import os\n", + "\n", + "from feast.feature_store import FeatureStore\n", + "\n", + "store = FeatureStore(os.environ['FEATURE_REPO'])\n", + "for data_source in store.registry.list_data_sources(store.project):\n", + " print(data_source.name)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "065128ef-6808-4c70-b5a7-dc5e3518e878", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.9", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}