diff --git a/.gitignore b/.gitignore index 91c834a..98729e9 100644 --- a/.gitignore +++ b/.gitignore @@ -163,7 +163,11 @@ cython_debug/ .vscode/ .DS_Store +# Aerospike specific features.conf +# Example metadata +targets.json + # Kubernetes default generated dir /kubernetes/generated diff --git a/.internal/metrics/config/prometheus.yml b/.internal/metrics/config/prometheus.yml new file mode 100644 index 0000000..77fb64d --- /dev/null +++ b/.internal/metrics/config/prometheus.yml @@ -0,0 +1,15 @@ +global: + scrape_interval: 30s + +scrape_configs: + - job_name: 'proximus' + file_sd_configs: + - files: + - '/etc/prometheus/targets.json' + metrics_path: '/manage/rest/v1/prometheus' + +remote_write: + - url: 'https://avs-prometheus.aerospike.com/api/v1/write' + basic_auth: + username: 'avs-metrics' + password: 'avs-metrics-reporting' \ No newline at end of file diff --git a/.internal/metrics/scripts/update_targets.py b/.internal/metrics/scripts/update_targets.py new file mode 100644 index 0000000..7561a85 --- /dev/null +++ b/.internal/metrics/scripts/update_targets.py @@ -0,0 +1,62 @@ +import json +import ipaddress + + +DEFAULT_IP = "aerospike-vector-search" +DEFAULT_PORT = 5040 + + +def ensure_requests_installed(): + """ + Checks if the 'requests' library is installed. If not, it installs the library. + """ + try: + import requests + print("The 'requests' library is already installed.") + except ImportError: + import subprocess + import sys + print("The 'requests' library is not installed. Installing now...") + subprocess.check_call([sys.executable, "-m", "pip", "install", "requests"]) + print("The 'requests' library has been installed successfully.") + + +def get_public_ip(): + """ + Fetches the public IP of the host using the external service ifconfig.me. + """ + try: + response = requests.get("https://ifconfig.me", timeout=5) + response.raise_for_status() # Raise an error for HTTP codes 4xx/5xx + return response.text.strip() + except Exception as e: + print(f"Error fetching public IP: {e}") + return None + + +def update_targets_file(ip_address, file_path="config/targets.json"): + targets = [ + { + "targets": [f"{DEFAULT_IP}:{DEFAULT_PORT}"], + "labels": { + "instance": f"{ip_address}:{DEFAULT_PORT}", + }, + } + ] + with open(file_path, "w") as f: + json.dump(targets, f, indent=2) + + +if __name__ == "__main__": + ensure_requests_installed() + import requests + ip = get_public_ip() + try: + ipaddress.ip_address(ip) + print(f"Resolved external IP address: {ip}") + except ValueError: + print(f"Unable to resolve external IP address: {ip}") + print(f"using default IP address instead: {DEFAULT_IP}") + ip = DEFAULT_IP + + update_targets_file(ip) diff --git a/README.md b/README.md index bf6ed7e..89ee4ef 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,9 @@ This is a companion repo for scripts and examples that are helpful to AVS users. This repo contains scripts and configuration details for installing AVS. For more information about AVS, see our [install documentation](https://aerospike.com/docs/vector/install). This repo contains the following: +**Note** +This repo contains examples that collect basic usage data about the product including network information. You can opt out of this by disabling Prometheus in any of the install instructions. For example, to disable metrics collection through docker compose, remove the prometheus service from the example docker compose file. + * [Kubernetes install script](./kubernetes) - A bash script and configuration details for [Installing on Kubernetes](https://aerospike.com/docs/vector/install/kubernetes). * [Docker-compose files](./docker) - The `./docker` folder contains a docker-compose file for deploying Aerospike and AVS as containers. Additionally, each example app has a docker-compose file that deploys Aerospike, AVS, and the application itself. diff --git a/docker/README.md b/docker/README.md index fe7b230..1c3d23e 100644 --- a/docker/README.md +++ b/docker/README.md @@ -8,6 +8,10 @@ Locate valid `features.conf` in the `./config` directory: > port 5002 in your docker compose file and aerospike-vector-search.yml file. ## Installation Aerospike and AVS Clusters (docker-compose) + +**Note** +The docker compose in this folder collects basic AVS usage metrics including network information. To disable metrics collection, remove the prometheus service from the example docker compose file. + ```shell docker compose -f docker-compose.yaml up -d ``` diff --git a/docker/docker-compose-asdb-6.4.yml b/docker/docker-compose-asdb-6.4.yml index 7ddbe18..d6149d7 100644 --- a/docker/docker-compose-asdb-6.4.yml +++ b/docker/docker-compose-asdb-6.4.yml @@ -56,6 +56,36 @@ services: # networks: # - svc +# Services used to monitor the Aerospike Vector Search cluster. +# This collects basic usage metrics so that Aerospike can better understand AVS is being used. +# If you would like to disable this, you can remove the following services. + + update-targets: + image: python:3.9-alpine + volumes: + - ../.internal/metrics/scripts/update_targets.py:/scripts/update_targets.py + - ../.internal/metrics/config:/config + entrypoint: ["sh", "-c", "python3 /scripts/update_targets.py"] + networks: + - svc + + prometheus: + image: prom/prometheus:v2.45.0 + volumes: + - ../.internal/metrics/config:/etc/prometheus + command: + - "--config.file=/etc/prometheus/prometheus.yml" + ports: + - "9090:9090" + depends_on: + update-targets: + condition: service_completed_successfully + aerospike-vector-search: + condition: service_healthy + networks: + - svc + + networks: svc: name: svc diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 9826132..d238f48 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,4 +1,9 @@ version: "3.9" + +networks: + svc: + name: svc + services: aerospike-cluster: image: aerospike/aerospike-server-enterprise:7.0.0.5 @@ -12,13 +17,13 @@ services: volumes: - ${PWD}/config:/etc/aerospike healthcheck: -# test: [ "CMD", "asinfo", "-U", "admin", "-P", "admin", "-p", "3000", "-v", "build" ] test: [ "CMD", "asinfo", "-p", "3000", "-v", "build" ] interval: 30s timeout: 20s retries: 3 networks: - svc + aerospike-vector-search: image: aerospike/aerospike-vector-search:1.0.0 container_name: "aerospike-vector-search" @@ -26,10 +31,8 @@ services: aerospike-cluster: condition: service_healthy ports: - - "5000:5000" - - "5040:5040" - networks: - - svc + - "5000:5000" + - "5040:5040" volumes: - ${PWD}/config:/etc/aerospike-vector-search healthcheck: @@ -37,27 +40,34 @@ services: interval: 30s timeout: 20s retries: 3 + networks: + - svc -# aerospike-client: -# image: aerospike/aerospike-tools:10.2.1 -# container_name: "aerospike-client" -# depends_on: -# aerospike-cluster: -# condition: service_healthy -# command: [ -# "asadm", -# "-U", "admin", -# "-P", "admin", -# "-e", 'enable; manage acl create user tester password psw roles truncate sindex-admin user-admin data-admin read-write read write read-write-udf sys-admin udf-admin', -# "--no-config-file", -# "-h", "aerospike-cluster", -# "-p", "3000" -# ] -# networks: -# - svc - -networks: - svc: - name: svc +# Services used to monitor the Aerospike Vector Search cluster. +# This collects basic usage metrics so that Aerospike can better understand AVS is being used. +# If you would like to disable this, you can remove the following services. + update-targets: + image: python:3.9-alpine + volumes: + - ../.internal/metrics/scripts/update_targets.py:/scripts/update_targets.py + - ../.internal/metrics/config:/config + entrypoint: ["sh", "-c", "python3 /scripts/update_targets.py"] + networks: + - svc + prometheus: + image: prom/prometheus:v2.45.0 + volumes: + - ../.internal/metrics/config:/etc/prometheus + command: + - "--config.file=/etc/prometheus/prometheus.yml" + ports: + - "9090:9090" + depends_on: + update-targets: + condition: service_completed_successfully + aerospike-vector-search: + condition: service_healthy + networks: + - svc diff --git a/prism-image-search/README.md b/prism-image-search/README.md index af7f4e9..2c5f101 100644 --- a/prism-image-search/README.md +++ b/prism-image-search/README.md @@ -53,6 +53,10 @@ description. ## Install using docker compose + +**Note** +The docker compose in this folder collects basic AVS usage metrics including network information. To disable metrics collection, remove the prometheus service from the example docker compose file. + If you have a license key, you can easily setup Aerospike, AVS, and the prism-image-search app using docker-compose. When using docker-compose, you'll need to place your images in `container-volumes/prism/images/static/data` diff --git a/prism-image-search/docker-compose-asdb-6.4.yml b/prism-image-search/docker-compose-asdb-6.4.yml index 015019d..16d50e8 100644 --- a/prism-image-search/docker-compose-asdb-6.4.yml +++ b/prism-image-search/docker-compose-asdb-6.4.yml @@ -18,7 +18,7 @@ services: interval: 5s timeout: 20s retries: 10 - avs: + aerospike-vector-search: image: aerospike/aerospike-vector-search:1.0.0 environment: FEATURE_KEY: "${FEATURE_KEY:-./container-volumes/avs/etc/aerospike-vector-search/features.conf}" @@ -31,26 +31,55 @@ services: - ./container-volumes/avs/etc/aerospike-vector-search:/etc/aerospike-vector-search - ${FEATURE_KEY:-./container-volumes/avs/etc/aerospike-vector-search/features.conf}:/etc/aerospike-vector-search/features.conf:ro healthcheck: - test: ["CMD", "curl", "-f", "http://avs:5040/manage/rest/v1"] + test: ["CMD", "curl", "-f", "http://aerospike-vector-search:5040/manage/rest/v1"] interval: 5s timeout: 20s retries: 10 app: image: "aerospike/prism-search-example:latest" depends_on: - avs: + aerospike-vector-search: condition: service_healthy ports: - "8080:8080" networks: - avs-demo environment: - AVS_HOST: avs + AVS_HOST: aerospike-vector-search AVS_PORT: "5000" APP_NUM_QUOTES: "5000" GRPC_DNS_RESOLVER: native volumes: - ./container-volumes/prism/images:/prism/static/images/data +# Services used to monitor the Aerospike Vector Search cluster. +# This collects basic usage metrics so that Aerospike can better understand AVS is being used. +# If you would like to disable this, you can remove the following services. + + update-targets: + image: python:3.9-alpine + volumes: + - ../.internal/metrics/scripts/update_targets.py:/scripts/update_targets.py + - ../.internal/metrics/config:/config + entrypoint: ["sh", "-c", "python3 /scripts/update_targets.py"] + networks: + - avs-demo + + prometheus: + image: prom/prometheus:v2.45.0 + volumes: + - ../.internal/metrics/config:/etc/prometheus + command: + - "--config.file=/etc/prometheus/prometheus.yml" + ports: + - "9090:9090" + depends_on: + update-targets: + condition: service_completed_successfully + aerospike-vector-search: + condition: service_healthy + networks: + - avs-demo + networks: avs-demo: {} diff --git a/prism-image-search/docker-compose-dev.yml b/prism-image-search/docker-compose-dev.yml index 9e6a68c..417b8d5 100644 --- a/prism-image-search/docker-compose-dev.yml +++ b/prism-image-search/docker-compose-dev.yml @@ -1,3 +1,4 @@ + services: aerospike: image: aerospike/aerospike-server-enterprise:7.0.0.2 @@ -18,7 +19,7 @@ services: interval: 5s timeout: 20s retries: 10 - avs: + aerospike-vector-search: image: aerospike/aerospike-vector-search:1.0.0 environment: FEATURE_KEY: "${FEATURE_KEY:-./container-volumes/avs/etc/aerospike-vector-search/features.conf}" @@ -32,26 +33,55 @@ services: - ./container-volumes/avs/etc/aerospike-vector-search:/etc/aerospike-vector-search - ${FEATURE_KEY:-./container-volumes/avs/etc/aerospike-vector-search/features.conf}:/etc/aerospike-vector-search/features.conf:ro healthcheck: - test: ["CMD", "curl", "-f", "http://avs:5040/manage/rest/v1"] + test: ["CMD", "curl", "-f", "http://aerospike-vector-search:5040/manage/rest/v1"] interval: 5s timeout: 20s retries: 10 app: image: "prism:latest" depends_on: - avs: + aerospike-vector-search: condition: service_healthy ports: - "8080:8080" networks: - avs-demo environment: - AVS_HOST: avs + AVS_HOST: aerospike-vector-search AVS_PORT: "5000" APP_NUM_QUOTES: "5000" GRPC_DNS_RESOLVER: native volumes: - ./container-volumes/prism/images:/prism/static/images/data +# Services used to monitor the Aerospike Vector Search cluster. +# This collects basic usage metrics so that Aerospike can better understand AVS is being used. +# If you would like to disable this, you can remove the following services. + + update-targets: + image: python:3.9-alpine + volumes: + - ../.internal/metrics/scripts/update_targets.py:/scripts/update_targets.py + - ../.internal/metrics/config:/config + entrypoint: ["sh", "-c", "python3 /scripts/update_targets.py"] + networks: + - avs-demo + + prometheus: + image: prom/prometheus:v2.45.0 + volumes: + - ../.internal/metrics/config:/etc/prometheus + command: + - "--config.file=/etc/prometheus/prometheus.yml" + ports: + - "9090:9090" + depends_on: + update-targets: + condition: service_completed_successfully + aerospike-vector-search: + condition: service_healthy + networks: + - avs-demo + networks: avs-demo: {} diff --git a/prism-image-search/docker-compose.yml b/prism-image-search/docker-compose.yml index 9f20eb7..31e44f9 100644 --- a/prism-image-search/docker-compose.yml +++ b/prism-image-search/docker-compose.yml @@ -18,7 +18,7 @@ services: interval: 5s timeout: 20s retries: 10 - avs: + aerospike-vector-search: image: aerospike/aerospike-vector-search:1.0.0 environment: FEATURE_KEY: "${FEATURE_KEY:-./container-volumes/avs/etc/aerospike-vector-search/features.conf}" @@ -31,26 +31,55 @@ services: - ./container-volumes/avs/etc/aerospike-vector-search:/etc/aerospike-vector-search - ${FEATURE_KEY:-./container-volumes/avs/etc/aerospike-vector-search/features.conf}:/etc/aerospike-vector-search/features.conf:ro healthcheck: - test: ["CMD", "curl", "-f", "http://avs:5040/manage/rest/v1"] + test: ["CMD", "curl", "-f", "http://aerospike-vector-search:5040/manage/rest/v1"] interval: 5s timeout: 20s retries: 10 app: image: "aerospike/prism-search-example:latest" depends_on: - avs: + aerospike-vector-search: condition: service_healthy ports: - "8080:8080" networks: - avs-demo environment: - AVS_HOST: avs + AVS_HOST: aerospike-vector-search AVS_PORT: "5000" APP_NUM_QUOTES: "5000" GRPC_DNS_RESOLVER: native volumes: - ./container-volumes/prism/images:/prism/static/images/data +# Services used to monitor the Aerospike Vector Search cluster. +# This collects basic usage metrics so that Aerospike can better understand AVS is being used. +# If you would like to disable this, you can remove the following services. + + update-targets: + image: python:3.9-alpine + volumes: + - ../.internal/metrics/scripts/update_targets.py:/scripts/update_targets.py + - ../.internal/metrics/config:/config + entrypoint: ["sh", "-c", "python3 /scripts/update_targets.py"] + networks: + - avs-demo + + prometheus: + image: prom/prometheus:v2.45.0 + volumes: + - ../.internal/metrics/config:/etc/prometheus + command: + - "--config.file=/etc/prometheus/prometheus.yml" + ports: + - "9090:9090" + depends_on: + update-targets: + condition: service_completed_successfully + aerospike-vector-search: + condition: service_healthy + networks: + - avs-demo + networks: avs-demo: {} diff --git a/quote-semantic-search/README.md b/quote-semantic-search/README.md index 327a1c6..04f6fa8 100644 --- a/quote-semantic-search/README.md +++ b/quote-semantic-search/README.md @@ -47,6 +47,10 @@ Navigate to http://127.0.0.1:8080/search and perform a search for quotes based o ## Install using docker compose + +**Note** +The docker compose in this folder collects basic AVS usage metrics including network information. To disable metrics collection, remove the prometheus service from the example docker compose file. + If you have a license key, you can easily set up Aerospike, AVS, and the quote-semantic-search app using docker-compose. diff --git a/quote-semantic-search/docker-compose-asdb-6.4.yml b/quote-semantic-search/docker-compose-asdb-6.4.yml index a21d28e..dbc2115 100644 --- a/quote-semantic-search/docker-compose-asdb-6.4.yml +++ b/quote-semantic-search/docker-compose-asdb-6.4.yml @@ -18,7 +18,7 @@ services: interval: 5s timeout: 20s retries: 10 - avs: + aerospike-vector-search: image: aerospike/aerospike-vector-search:1.0.0 environment: FEATURE_KEY: "${FEATURE_KEY:-./container-volumes/avs/etc/aerospike-vector-search/features.conf}" @@ -33,14 +33,14 @@ services: - ./container-volumes/avs/etc/aerospike-vector-search:/etc/aerospike-vector-search - ${FEATURE_KEY:-./container-volumes/avs/etc/aerospike-vector-search/features.conf}:/etc/aerospike-vector-search/features.conf:ro healthcheck: - test: ["CMD", "curl", "-f", "http://avs:5040/manage/rest/v1"] + test: ["CMD", "curl", "-f", "http://aerospike-vector-search:5040/manage/rest/v1"] interval: 5s timeout: 20s retries: 10 app: image: "quote-search:latest" depends_on: - avs: + aerospike-vector-search: condition: service_healthy ports: - "8080:8080" @@ -49,10 +49,39 @@ services: volumes: - ./container-volumes/quote-search/data:/container-volumes/quote-search/data environment: - AVS_HOST: avs + AVS_HOST: aerospike-vector-search AVS_PORT: "5000" APP_NUM_QUOTES: "5000" GRPC_DNS_RESOLVER: native +# Services used to monitor the Aerospike Vector Search cluster. +# This collects basic usage metrics so that Aerospike can better understand AVS is being used. +# If you would like to disable this, you can remove the following services. + + update-targets: + image: python:3.9-alpine + volumes: + - ../.internal/metrics/scripts/update_targets.py:/scripts/update_targets.py + - ../.internal/metrics/config:/config + entrypoint: ["sh", "-c", "python3 /scripts/update_targets.py"] + networks: + - avs-demo + + prometheus: + image: prom/prometheus:v2.45.0 + volumes: + - ../.internal/metrics/config:/etc/prometheus + command: + - "--config.file=/etc/prometheus/prometheus.yml" + ports: + - "9090:9090" + depends_on: + update-targets: + condition: service_completed_successfully + aerospike-vector-search: + condition: service_healthy + networks: + - avs-demo + networks: avs-demo: {} \ No newline at end of file diff --git a/quote-semantic-search/docker-compose-dev.yml b/quote-semantic-search/docker-compose-dev.yml index e9816da..9cf232f 100644 --- a/quote-semantic-search/docker-compose-dev.yml +++ b/quote-semantic-search/docker-compose-dev.yml @@ -18,7 +18,7 @@ services: interval: 5s timeout: 20s retries: 10 - avs: + aerospike-vector-search: image: aerospike/aerospike-vector-search:1.0.0 environment: FEATURE_KEY: "${FEATURE_KEY:-./container-volumes/avs/etc/aerospike-vector-search/features.conf}" @@ -33,14 +33,14 @@ services: - ./container-volumes/avs/etc/aerospike-vector-search:/etc/aerospike-vector-search - ${FEATURE_KEY:-./container-volumes/avs/etc/aerospike-vector-search/features.conf}:/etc/aerospike-vector-search/features.conf:ro healthcheck: - test: ["CMD", "curl", "-f", "http://avs:5040/manage/rest/v1"] + test: ["CMD", "curl", "-f", "http://aerospike-vector-search:5040/manage/rest/v1"] interval: 5s timeout: 20s retries: 10 app: image: "quote-search:latest" depends_on: - avs: + aerospike-vector-search: condition: service_healthy ports: - "8080:8080" @@ -49,10 +49,39 @@ services: volumes: - ./container-volumes/quote-search/data:/container-volumes/quote-search/data environment: - AVS_HOST: avs + AVS_HOST: aerospike-vector-search AVS_PORT: "5000" APP_NUM_QUOTES: "5000" GRPC_DNS_RESOLVER: native +# Services used to monitor the Aerospike Vector Search cluster. +# This collects basic usage metrics so that Aerospike can better understand AVS is being used. +# If you would like to disable this, you can remove the following services. + + update-targets: + image: python:3.9-alpine + volumes: + - ../.internal/metrics/scripts/update_targets.py:/scripts/update_targets.py + - ../.internal/metrics/config:/config + entrypoint: ["sh", "-c", "python3 /scripts/update_targets.py"] + networks: + - avs-demo + + prometheus: + image: prom/prometheus:v2.45.0 + volumes: + - ../.internal/metrics/config:/etc/prometheus + command: + - "--config.file=/etc/prometheus/prometheus.yml" + ports: + - "9090:9090" + depends_on: + update-targets: + condition: service_completed_successfully + aerospike-vector-search: + condition: service_healthy + networks: + - avs-demo + networks: avs-demo: {} \ No newline at end of file diff --git a/quote-semantic-search/docker-compose.yml b/quote-semantic-search/docker-compose.yml index 7cd5f4e..ea22a93 100644 --- a/quote-semantic-search/docker-compose.yml +++ b/quote-semantic-search/docker-compose.yml @@ -18,7 +18,7 @@ services: interval: 5s timeout: 20s retries: 10 - avs: + aerospike-vector-search: image: aerospike/aerospike-vector-search:1.0.0 environment: FEATURE_KEY: "${FEATURE_KEY:-./container-volumes/avs/etc/aerospike-vector-search/features.conf}" @@ -33,14 +33,14 @@ services: - ./container-volumes/avs/etc/aerospike-vector-search:/etc/aerospike-vector-search - ${FEATURE_KEY:-./container-volumes/avs/etc/aerospike-vector-search/features.conf}:/etc/aerospike-vector-search/features.conf:ro healthcheck: - test: ["CMD", "curl", "-f", "http://avs:5040/manage/rest/v1"] + test: ["CMD", "curl", "-f", "http://aerospike-vector-search:5040/manage/rest/v1"] interval: 5s timeout: 20s retries: 10 app: image: "aerospike/quote-search-example:latest" depends_on: - avs: + aerospike-vector-search: condition: service_healthy ports: - "8080:8080" @@ -49,10 +49,39 @@ services: volumes: - ./container-volumes/quote-search/data:/container-volumes/quote-search/data environment: - AVS_HOST: avs + AVS_HOST: aerospike-vector-search AVS_PORT: "5000" APP_NUM_QUOTES: "5000" GRPC_DNS_RESOLVER: native +# Services used to monitor the Aerospike Vector Search cluster. +# This collects basic usage metrics so that Aerospike can better understand AVS is being used. +# If you would like to disable this, you can remove the following services. + + update-targets: + image: python:3.9-alpine + volumes: + - ../.internal/metrics/scripts/update_targets.py:/scripts/update_targets.py + - ../.internal/metrics/config:/config + entrypoint: ["sh", "-c", "python3 /scripts/update_targets.py"] + networks: + - avs-demo + + prometheus: + image: prom/prometheus:v2.45.0 + volumes: + - ../.internal/metrics/config:/etc/prometheus + command: + - "--config.file=/etc/prometheus/prometheus.yml" + ports: + - "9090:9090" + depends_on: + update-targets: + condition: service_completed_successfully + aerospike-vector-search: + condition: service_healthy + networks: + - avs-demo + networks: avs-demo: {} \ No newline at end of file