From d3eedb5e716cdc3299021f5e0c75d678c7c85da0 Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Wed, 18 Oct 2023 11:19:43 +0200 Subject: [PATCH] feat: add prometheus metrics (#3640) --- .dockerignore | 1 + Dockerfile | 4 +++- gunicorn.conf.py | 14 ++++++++++++ poetry.lock | 41 +++++++++++++++++++++++++++++----- pyproject.toml | 2 ++ renku/ui/cli/service.py | 2 ++ renku/ui/service/entrypoint.py | 5 +++++ 7 files changed, 62 insertions(+), 7 deletions(-) create mode 100644 gunicorn.conf.py diff --git a/.dockerignore b/.dockerignore index c44c0349b2..f9dda275b8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,4 +8,5 @@ !.git !.gitignore !Makefile +!gunicorn.conf.py .git/config diff --git a/Dockerfile b/Dockerfile index 3fb5d9d40c..76bb02e244 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN apt-get install --no-install-recommends -y build-essential && \ # time the code changes # set the BUILD_CORE_SERVICE to non null to install additional service dependencies ARG BUILD_CORE_SERVICE -COPY pyproject.toml poetry.lock README.rst CHANGES.rst Makefile /code/renku/ +COPY pyproject.toml poetry.lock README.rst CHANGES.rst Makefile gunicorn.conf.py /code/renku/ COPY .git /code/renku/.git COPY renku /code/renku/renku WORKDIR /code/renku @@ -46,6 +46,7 @@ RUN addgroup -gid 1000 shuhitsu && \ if [ -n "${BUILD_CORE_SERVICE}" ]; then mkdir /svc && chown shuhitsu:shuhitsu /svc ; fi COPY --from=builder /code/renku /code/renku +WORKDIR /code/renku ENV PATH="${PATH}:/code/renku/.venv/bin" # shuhitsu (執筆): The "secretary" of the renga, as it were, who is responsible for @@ -55,5 +56,6 @@ USER shuhitsu ENV RENKU_SVC_NUM_WORKERS 4 ENV RENKU_SVC_NUM_THREADS 8 ENV RENKU_DISABLE_VERSION_CHECK=1 +ENV PROMETHEUS_MULTIPROC_DIR /tmp ENTRYPOINT ["tini", "-g", "--", "renku"] diff --git a/gunicorn.conf.py b/gunicorn.conf.py new file mode 100644 index 0000000000..6f524217ff --- /dev/null +++ b/gunicorn.conf.py @@ -0,0 +1,14 @@ +"""Gunicorn Configuration.""" +import os + +from prometheus_flask_exporter.multiprocess import GunicornPrometheusMetrics + + +def when_ready(server): + """Run metrics server on separate port.""" + GunicornPrometheusMetrics.start_http_server_when_ready(int(os.getenv("METRICS_PORT", "8765"))) + + +def child_exit(server, worker): + """Properly exit when metrics server stops.""" + GunicornPrometheusMetrics.mark_process_dead_on_child_exit(worker.pid) diff --git a/poetry.lock b/poetry.lock index 8c9fdcea1d..8b13e0673f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "addict" @@ -749,8 +749,8 @@ files = [ [package.dependencies] "ruamel.yaml" = [ {version = ">=0.16.0,<0.18", markers = "python_version >= \"3.10\""}, - {version = ">=0.15.78,<0.18", markers = "python_version >= \"3.8\""}, - {version = ">=0.15.98,<0.18", markers = "python_version >= \"3.9\""}, + {version = ">=0.15.98,<0.18", markers = "python_version >= \"3.9\" and python_version < \"3.10\""}, + {version = ">=0.15.78,<0.18", markers = "python_version >= \"3.8\" and python_version < \"3.9\""}, ] schema-salad = "*" setuptools = "*" @@ -1040,7 +1040,7 @@ files = [ name = "flask" version = "2.2.5" description = "A simple framework for building complex web applications." -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "Flask-2.2.5-py3-none-any.whl", hash = "sha256:58107ed83443e86067e41eff4631b058178191a355886f8e479e347fa1285fdf"}, @@ -1357,7 +1357,7 @@ requirements-deprecated-finder = ["pip-api", "pipreqs"] name = "itsdangerous" version = "2.1.2" description = "Safely pass data to untrusted environments and back." -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "itsdangerous-2.1.2-py3-none-any.whl", hash = "sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44"}, @@ -2390,6 +2390,35 @@ wcwidth = "*" [package.extras] tests = ["pytest", "pytest-cov", "pytest-lazy-fixture"] +[[package]] +name = "prometheus-client" +version = "0.17.1" +description = "Python client for the Prometheus monitoring system." +optional = false +python-versions = ">=3.6" +files = [ + {file = "prometheus_client-0.17.1-py3-none-any.whl", hash = "sha256:e537f37160f6807b8202a6fc4764cdd19bac5480ddd3e0d463c3002b34462101"}, + {file = "prometheus_client-0.17.1.tar.gz", hash = "sha256:21e674f39831ae3f8acde238afd9a27a37d0d2fb5a28ea094f0ce25d2cbf2091"}, +] + +[package.extras] +twisted = ["twisted"] + +[[package]] +name = "prometheus-flask-exporter" +version = "0.22.4" +description = "Prometheus metrics exporter for Flask" +optional = false +python-versions = "*" +files = [ + {file = "prometheus_flask_exporter-0.22.4-py3-none-any.whl", hash = "sha256:e130179c26d5a9b903c12c0d8826127ae491b04b298cae0b92b98677dcf2c06f"}, + {file = "prometheus_flask_exporter-0.22.4.tar.gz", hash = "sha256:959b69f1e740b6736ea53fe5f28dc2ab6229b2ebeade6582b3dbb5d74c7d58e4"}, +] + +[package.dependencies] +flask = "*" +prometheus-client = "*" + [[package]] name = "prov" version = "1.5.1" @@ -4582,4 +4611,4 @@ service = ["apispec", "apispec-oneofschema", "apispec-webframeworks", "circus", [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<3.12" -content-hash = "a75f7197bf53bde1db069a16c1192aa982dbdf84d6c864bb032f8f764c34f154" +content-hash = "1649b695abc65d476919f07e320a2675210a1232d59897783aa66359f0163af3" diff --git a/pyproject.toml b/pyproject.toml index e4633b2b7c..a2de1b7a46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -114,6 +114,7 @@ redis = { version = ">=3.5.3,<4.6.0,!=4.5.5", optional = true } rq = { version = "==1.15.0", optional = true } sentry-sdk = { version = ">=1.5.11,<1.26.0", extras = ["flask"], optional = true } walrus = { version = ">=0.8.2,<0.10.0", optional = true } +prometheus-flask-exporter = "^0.22.4" [tool.poetry.group.dev.dependencies] black = "==23.1.0" @@ -301,6 +302,7 @@ module = [ "pexpect", "PIL", "pluggy", + "prometheus_flask_exporter.*", "psutil", "pyld", "pyshacl", diff --git a/renku/ui/cli/service.py b/renku/ui/cli/service.py index b7ec26a618..22948f51f2 100644 --- a/renku/ui/cli/service.py +++ b/renku/ui/cli/service.py @@ -50,6 +50,8 @@ def run_api(addr="0.0.0.0", port=8080, timeout=600): "gunicorn", "renku.ui.service.entrypoint:app", loading_opt, + "-c", + "gunicorn.conf.py", "-b", f"{addr}:{port}", "--timeout", diff --git a/renku/ui/service/entrypoint.py b/renku/ui/service/entrypoint.py index 1571c5ebda..5c67c94505 100644 --- a/renku/ui/service/entrypoint.py +++ b/renku/ui/service/entrypoint.py @@ -22,10 +22,12 @@ import sentry_sdk from flask import Flask, Response, jsonify, request, url_for from jwt import InvalidTokenError +from prometheus_flask_exporter.multiprocess import GunicornPrometheusMetrics from sentry_sdk.integrations.flask import FlaskIntegration from sentry_sdk.integrations.redis import RedisIntegration from sentry_sdk.integrations.rq import RqIntegration +from renku.core.util.util import is_test_session_running from renku.ui.service.cache import cache from renku.ui.service.config import CACHE_DIR, MAX_CONTENT_LENGTH, SENTRY_ENABLED, SENTRY_SAMPLERATE, SERVICE_PREFIX from renku.ui.service.errors import ( @@ -74,6 +76,9 @@ def create_app(custom_exceptions=True): app.config["cache"] = cache + if not is_test_session_running(): + GunicornPrometheusMetrics(app) + build_routes(app) @app.route(SERVICE_PREFIX)