diff --git a/notebook/base/handlers.py b/notebook/base/handlers.py index 430ff0f0ed..fb8cbe5665 100755 --- a/notebook/base/handlers.py +++ b/notebook/base/handlers.py @@ -28,6 +28,7 @@ from jinja2 import TemplateNotFound from tornado import web, gen, escape, httputil from tornado.log import app_log +import prometheus_client from notebook._sysinfo import get_sys_info @@ -809,6 +810,16 @@ def get(self): url = sep.join([self._url, self.request.query]) self.redirect(url, permanent=self._permanent) +class PrometheusMetricsHandler(IPythonHandler): + """ + Return prometheus metrics for this notebook server + """ + @web.authenticated + def get(self): + self.set_header('Content-Type', prometheus_client.CONTENT_TYPE_LATEST) + self.write(prometheus_client.generate_latest(prometheus_client.REGISTRY)) + + #----------------------------------------------------------------------------- # URL pattern fragments for re-use #----------------------------------------------------------------------------- @@ -825,4 +836,5 @@ def get(self): (r".*/", TrailingSlashHandler), (r"api", APIVersionHandler), (r'/(robots\.txt|favicon\.ico)', web.StaticFileHandler), + (r'/metrics', PrometheusMetricsHandler) ] diff --git a/notebook/log.py b/notebook/log.py index dab330cb48..64b35d811d 100644 --- a/notebook/log.py +++ b/notebook/log.py @@ -7,6 +7,7 @@ import json from tornado.log import access_log +from .metrics import prometheus_log_method def log_request(handler): """log a bit more information about each request than tornado's default @@ -45,4 +46,4 @@ def log_request(handler): # log all headers if it caused an error log_method(json.dumps(dict(request.headers), indent=2)) log_method(msg.format(**ns)) - + prometheus_log_method(handler) diff --git a/notebook/metrics.py b/notebook/metrics.py new file mode 100644 index 0000000000..24a08d8c88 --- /dev/null +++ b/notebook/metrics.py @@ -0,0 +1,37 @@ +""" +Prometheus metrics exported by Jupyter Notebook Server + +Read https://prometheus.io/docs/practices/naming/ for naming +conventions for metrics & labels. +""" + +from prometheus_client import Histogram + +# This is a fairly standard name for HTTP duration latency reporting +HTTP_REQUEST_DURATION_SECONDS = Histogram( + 'http_request_duration_seconds', + 'duration in seconds for all HTTP requests', + ['method', 'handler', 'status_code'], +) + +def prometheus_log_method(handler): + """ + Tornado log handler for recording RED metrics. + + We record the following metrics: + Rate - the number of requests, per second, your services are serving. + Errors - the number of failed requests per second. + Duration - The amount of time each request takes expressed as a time interval. + + We use a fully qualified name of the handler as a label, + rather than every url path to reduce cardinality. + + This function should be either the value of or called from a function + that is the 'log_function' tornado setting. This makes it get called + at the end of every request, allowing us to record the metrics we need. + """ + HTTP_REQUEST_DURATION_SECONDS.labels( + method=handler.request.method, + handler='{}.{}'.format(handler.__class__.__module__, type(handler).__name__), + status_code=handler.get_status() + ).observe(handler.request.request_time()) diff --git a/setup.py b/setup.py index a29cf79594..bfaba44877 100755 --- a/setup.py +++ b/setup.py @@ -87,7 +87,8 @@ 'nbconvert', 'ipykernel', # bless IPython kernel for now 'Send2Trash', - 'terminado>=0.8.1' + 'terminado>=0.8.1', + 'prometheus_client' ], extras_require = { 'test:python_version == "2.7"': ['mock'],