From 040d1c1a98e5d604a21348b104f023d29f9c8489 Mon Sep 17 00:00:00 2001 From: Arnaud-D <35631001+Arnaud-D@users.noreply.github.com> Date: Sat, 20 Jan 2024 19:55:21 +0100 Subject: [PATCH] =?UTF-8?q?Int=C3=A8gre=20le=20code=20de=20django=5Fmunin?= =?UTF-8?q?=20=C3=A0=20notre=20d=C3=A9p=C3=B4t?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- django_munin/LICENSE.md | 30 +++++ django_munin/README.md | 113 ++++++++++++++++++ django_munin/__init__.py | 0 django_munin/munin/__init__.py | 0 django_munin/munin/helpers.py | 24 ++++ django_munin/munin/migrations/0001_initial.py | 9 ++ django_munin/munin/migrations/0002_test.py | 21 ++++ django_munin/munin/migrations/__init__.py | 0 django_munin/munin/models.py | 5 + django_munin/munin/urls.py | 11 ++ django_munin/munin/views.py | 65 ++++++++++ django_munin/plugins/django.py | 30 +++++ requirements.txt | 1 - zds/munin/tests.py | 27 +++++ zds/munin/urls.py | 15 +-- zds/munin/views.py | 2 +- zds/settings/abstract_base/django.py | 2 +- zds/urls.py | 3 +- 18 files changed, 346 insertions(+), 12 deletions(-) create mode 100644 django_munin/LICENSE.md create mode 100644 django_munin/README.md create mode 100644 django_munin/__init__.py create mode 100644 django_munin/munin/__init__.py create mode 100644 django_munin/munin/helpers.py create mode 100644 django_munin/munin/migrations/0001_initial.py create mode 100644 django_munin/munin/migrations/0002_test.py create mode 100644 django_munin/munin/migrations/__init__.py create mode 100644 django_munin/munin/models.py create mode 100644 django_munin/munin/urls.py create mode 100644 django_munin/munin/views.py create mode 100755 django_munin/plugins/django.py create mode 100644 zds/munin/tests.py diff --git a/django_munin/LICENSE.md b/django_munin/LICENSE.md new file mode 100644 index 0000000000..f4726b82b7 --- /dev/null +++ b/django_munin/LICENSE.md @@ -0,0 +1,30 @@ +This directory is based on the package django-munin (http://github.com/ccnmtl/django-munin/) +by Andreas Pearson (anders@columbia.edu). + +It is forked from the following commit : +https://github.com/ccnmtl/django-munin/commit/3675b26a963edbe075abc1947ce9f7befcfcb939 + +The original copyright notice and license terms are quoted below. + +> Copyright (c) 2011, Columbia Center For New Media Teaching And Learning (CCNMTL) +> All rights reserved. +> Redistribution and use in source and binary forms, with or without +> modification, are permitted provided that the following conditions are met: +> * Redistributions of source code must retain the above copyright +> notice, this list of conditions and the following disclaimer. +> * Redistributions in binary form must reproduce the above copyright +> notice, this list of conditions and the following disclaimer in the +> documentation and/or other materials provided with the distribution. +> * Neither the name of the CCNMTL nor the +> names of its contributors may be used to endorse or promote products +> derived from this software without specific prior written permission. +> THIS SOFTWARE IS PROVIDED BY CCNMTL ``AS IS'' AND ANY +> EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +> WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +> DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +> DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +> (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +> LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +> ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +> (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +> SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/django_munin/README.md b/django_munin/README.md new file mode 100644 index 0000000000..b107f5a514 --- /dev/null +++ b/django_munin/README.md @@ -0,0 +1,113 @@ +# django-munin + +This is a Django application to make it a bit simpler to use +[Munin](http://munin-monitoring.org/) to monitor various metrics +for your Django app. + +First, it includes a munin plugin that you can symlink into +`/etc/munin/plugins/` and point at your django application and it will +gather data for munin to graph. Second, it contains a couple views +that return some very basic information about the state of your app: +database performance, number of users, number of sessions, etc. Third, +it provides a decorator to make it simple to expose your own custom +metrics to Munin. + +## Installing + +Install `django-munin` into your python path with the usual `pip install` +or whatever you are doing. Then add `munin` to your `INSTALLED_APPS` and +run `manage.py syncdb` (it just needs to set up one database table +that it will use for performance testing). + +To access the included basic views, add the following pattern to your +`urls.py`: + + ('^munin/',include('munin.urls')), + +The views available there are then going to be at: + +* `munin/db_performance/` (milliseconds to perform insert/select/delete operations) +* `munin/total_users/` (total number of Users) +* `munin/active_users/` (number of users logged in in the last hour) +* `munin/total_sessions/` (total number of sessions) +* `munin/active_sessions/` (number of sessions that are not expired) + +Those were the only metrics I could think of that would be potentially +useful on just about any Django app and were likely to always be +available. + +(I'm going to assume that you are already a pro at configuring +Munin. If not, go get on that. Munin is very cool) + +Next, copy `plugins/django.py` into your `/usr/share/munin/plugins/` +directory. + +For each metric that you want Munin to monitor, make a symlink in +`/etc/munin/plugins/` to `/usr/share/munin/plugins/django.py` with an +appropriate name. Eg, to monitor all five of the included ones (as +root, probably): + + $ ln -s /usr/share/munin/plugins/django.py /etc/munin/plugins/myapp_db_performance + $ ln -s /usr/share/munin/plugins/django.py /etc/munin/plugins/myapp_total_users + $ ln -s /usr/share/munin/plugins/django.py /etc/munin/plugins/myapp_active_users + $ ln -s /usr/share/munin/plugins/django.py /etc/munin/plugins/myapp_total_sessions + $ ln -s /usr/share/munin/plugins/django.py /etc/munin/plugins/myapp_active_sessions + +You then need to configure each of them in +`/etc/munin/plugin-conf.d/munin-node` + +For each, give it a stanza with `env.url` and `graph_category` set. To +continue the above, you'd add something like: + + [myapp_db_performance] + env.url http://example.com/munin/db_performance/ + env.graph_category myapp + + [myapp_total_users] + env.url http://example.com/munin/total_users/ + env.graph_category myapp + + [myapp_active_users] + env.url http://example.com/munin/active_users/ + env.graph_category myapp + + [myapp_total_sessions] + env.url http://example.com/munin/total_sessions/ + env.graph_category myapp + + [myapp_active_sessions] + env.url http://example.com/munin/active_sessions/ + env.graph_category myapp + +If your HTTP server require Basic Authentication, you can add login and password +as parameters: + + [myapp_active_sessions] + env.url http://example.com/munin/active_sessions/ + env.graph_category myapp + env.login mylogin + env.password mypassword + +Restart your Munin node, and it should start collecting and graphing +that data. + +## Custom munin views + +Those are pretty generic metrics though and the real power of this +application is that you can easily expose your own custom +metrics. Basically, anything that you can calculate in the context of +a Django view in your application, you can easily expose to Munin. + +`django-munin` includes a `@muninview` decorator that lets you write a +regular django view that returns a list of `(key,value)` tuples and it +will expose those to that `django.py` munin plugin for easy graphing. + +The `@muninview` decorator takes a `config` parameter, which is just a +string of munin config directives. You'll want to put stuff like +`graph_title`, `graph_vlabel`, and `graph_info` there. Possibly +`graph_category` too (if you include it there, remove it from the munin +plugin conf stanza). The view function that it wraps then just needs +to return a list of tuples. + +The simplest way to get a feel for how this works is to look at how +the included views were written. So check out [munin/views.py](https://github.com/ccnmtl/django-munin/blob/master/munin/views.py). diff --git a/django_munin/__init__.py b/django_munin/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/django_munin/munin/__init__.py b/django_munin/munin/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/django_munin/munin/helpers.py b/django_munin/munin/helpers.py new file mode 100644 index 0000000000..84a386762c --- /dev/null +++ b/django_munin/munin/helpers.py @@ -0,0 +1,24 @@ +from django.http import HttpResponse + + +class muninview: + """decorator to make it simpler to write munin views""" + + def __init__(self, config=""): + self.config = config + + def __call__(self, func): + def rendered_func(request, *args, **kwargs): + tuples = func(request, *args, **kwargs) + if "autoconfig" in request.GET: + return HttpResponse("yes") + if "config" in request.GET: + rows = ["{}.label {}".format(t[0].replace(" ", "_"), t[0]) for t in tuples] + return HttpResponse("\n".join([self.config] + rows)) + if type(tuples) == type([]): + rows = ["{} {}".format(t[0].replace(" ", "_"), str(t[1])) for t in tuples] + return HttpResponse("\n".join(rows)) + else: + return tuples + + return rendered_func diff --git a/django_munin/munin/migrations/0001_initial.py b/django_munin/munin/migrations/0001_initial.py new file mode 100644 index 0000000000..2abc607f12 --- /dev/null +++ b/django_munin/munin/migrations/0001_initial.py @@ -0,0 +1,9 @@ +# Generated by Django 1.9.8 on 2016-12-05 01:52 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [] + + operations = [] diff --git a/django_munin/munin/migrations/0002_test.py b/django_munin/munin/migrations/0002_test.py new file mode 100644 index 0000000000..a631c37707 --- /dev/null +++ b/django_munin/munin/migrations/0002_test.py @@ -0,0 +1,21 @@ +# Generated by Django 1.9.8 on 2016-12-05 01:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [ + ("munin", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="Test", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("name", models.CharField(max_length=256)), + ], + ), + ] diff --git a/django_munin/munin/migrations/__init__.py b/django_munin/munin/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/django_munin/munin/models.py b/django_munin/munin/models.py new file mode 100644 index 0000000000..c36aa7cfc7 --- /dev/null +++ b/django_munin/munin/models.py @@ -0,0 +1,5 @@ +from django.db import models + + +class Test(models.Model): + name = models.CharField(max_length=256) diff --git a/django_munin/munin/urls.py b/django_munin/munin/urls.py new file mode 100644 index 0000000000..85626fed86 --- /dev/null +++ b/django_munin/munin/urls.py @@ -0,0 +1,11 @@ +from django.urls import path + +from .views import total_users, active_users, total_sessions, active_sessions, db_performance + +urlpatterns = [ + path("total_users/", total_users, name="total-users"), + path("active_users/", active_users, name="active-users"), + path("total_sessions/", total_sessions, name="total-sessions"), + path("active_sessions/", active_sessions, name="active-sessions"), + path("db_performance/", db_performance, name="db-performance"), +] diff --git a/django_munin/munin/views.py b/django_munin/munin/views.py new file mode 100644 index 0000000000..e5982aac7b --- /dev/null +++ b/django_munin/munin/views.py @@ -0,0 +1,65 @@ +from datetime import datetime +from datetime import timedelta +import time +from django.contrib.sessions.models import Session +from django.contrib.auth import get_user_model +from .helpers import muninview +from .models import Test + + +User = get_user_model() + + +@muninview( + config="""graph_title Total Users +graph_vlabel users""" +) +def total_users(request): + return [("users", User.objects.all().count())] + + +@muninview( + config="""graph_title Active Users +graph_vlabel users +graph_info Number of users logged in during the last hour""" +) +def active_users(request): + hour_ago = datetime.now() - timedelta(hours=1) + return [("users", User.objects.filter(last_login__gt=hour_ago).count())] + + +@muninview( + config="""graph_title Total Sessions +graph_vlabel sessions""" +) +def total_sessions(request): + return [("sessions", Session.objects.all().count())] + + +@muninview( + config="""graph_title Active Sessions +graph_vlabel sessions""" +) +def active_sessions(request): + return [("sessions", Session.objects.filter(expire_date__gt=datetime.now()).count())] + + +@muninview( + config="""graph_title DB performance +graph_vlabel milliseconds +graph_info performance of simple insert/select/delete operations""" +) +def db_performance(request): + start = time.time() + t = Test.objects.create(name="inserting at %f" % start) + end = time.time() + insert = end - start + start = time.time() + t2 = Test.objects.get(id=t.id) + end = time.time() + select = end - start + start = time.time() + t2.delete() + end = time.time() + delete = end - start + return [("insert", 1000 * insert), ("select", 1000 * select), ("delete", 1000 * delete)] diff --git a/django_munin/plugins/django.py b/django_munin/plugins/django.py new file mode 100755 index 0000000000..50f8d3efae --- /dev/null +++ b/django_munin/plugins/django.py @@ -0,0 +1,30 @@ +import sys +import urllib.request +import os +import base64 + +url = os.environ["url"] +category = os.environ.get("graph_category", "") +login = os.environ.get("login", "") +password = os.environ.get("password", "") +base64string = base64.encodestring(f"{login}:{password}").replace("\n", "") + +if len(sys.argv) == 2: + url = url + "?" + sys.argv[1] + "=1" + request = urllib.request.Request(url) + if login != "" and password != "": + request.add_header("Authorization", "Basic %s" % base64string) + print(urllib.request.urlopen(request).read()) + # they can set the category in the config + if category != "": + print("graph_category " + category) +else: + request = urllib.request.Request(url) + if login != "" and password != "": + request.add_header("Authorization", "Basic %s" % base64string) + data = urllib.request.urlopen(request).readlines() + for line in data: + parts = line.split(" ") + label = parts[0] + value = " ".join(parts[1:]) + print(label + ".value " + value) diff --git a/requirements.txt b/requirements.txt index 594c8b6f3a..ff9f0213f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,6 @@ social-auth-app-django==5.4.0 beautifulsoup4==4.12.2 django-crispy-forms==1.14.0 django-model-utils==4.3.1 -django-munin==0.2.1 django-recaptcha==4.0.0 Django==3.2.23 easy-thumbnails[svg]==2.8.5 diff --git a/zds/munin/tests.py b/zds/munin/tests.py new file mode 100644 index 0000000000..4c8b67bf77 --- /dev/null +++ b/zds/munin/tests.py @@ -0,0 +1,27 @@ +from django.test import TestCase +from django.urls import reverse + + +class Munin(TestCase): + def setUp(self): + base_names = [ + "base:total-users", + "base:active-users", + "base:total-sessions", + "base:active-sessions", + "base:db-performance", + "total-topics", + "total-posts", + "total-mp", + "total-tutorial", + "total-articles", + "total-opinions", + ] + self.routes = [f"munin:{base_name}" for base_name in base_names] + + def test_routes(self): + for route in self.routes: + with self.subTest(msg=route): + url = reverse(route) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) diff --git a/zds/munin/urls.py b/zds/munin/urls.py index 3784203d67..ea54720cee 100644 --- a/zds/munin/urls.py +++ b/zds/munin/urls.py @@ -1,13 +1,14 @@ -from django.urls import path +from django.urls import path, include from zds.munin.views import total_topics, total_posts, total_mps, total_tutorials, total_articles, total_opinions urlpatterns = [ - path("total_topics/", total_topics, name="total_topics"), - path("total_posts/", total_posts, name="total_posts"), - path("total_mps/", total_mps, name="total_mp"), - path("total_tutorials/", total_tutorials, name="total_tutorial"), - path("total_articles/", total_articles, name="total_articles"), - path("total_opinions/", total_opinions, name="total_opinions"), + path("", include(("django_munin.munin.urls", "base"))), + path("total_topics/", total_topics, name="total-topics"), + path("total_posts/", total_posts, name="total-posts"), + path("total_mps/", total_mps, name="total-mp"), + path("total_tutorials/", total_tutorials, name="total-tutorial"), + path("total_articles/", total_articles, name="total-articles"), + path("total_opinions/", total_opinions, name="total-opinions"), ] diff --git a/zds/munin/views.py b/zds/munin/views.py index eb5d745587..49ebf1568c 100644 --- a/zds/munin/views.py +++ b/zds/munin/views.py @@ -1,4 +1,4 @@ -from munin.helpers import muninview +from django_munin.munin.helpers import muninview from zds.forum.models import Topic, Post from zds.mp.models import PrivateTopic, PrivatePost from zds.tutorialv2.models.database import PublishableContent, ContentReaction diff --git a/zds/settings/abstract_base/django.py b/zds/settings/abstract_base/django.py index 9aad646410..d63ab323b0 100644 --- a/zds/settings/abstract_base/django.py +++ b/zds/settings/abstract_base/django.py @@ -156,7 +156,6 @@ "easy_thumbnails", "easy_thumbnails.optimize", "crispy_forms", - "munin", "social_django", "rest_framework", "drf_yasg", @@ -164,6 +163,7 @@ "corsheaders", "oauth2_provider", "django_recaptcha", + "django_munin.munin", # Apps DB tables are created in THIS order by default # --> Order is CRITICAL to properly handle foreign keys "zds.utils", diff --git a/zds/urls.py b/zds/urls.py index 293857d02d..4e514144ba 100644 --- a/zds/urls.py +++ b/zds/urls.py @@ -89,11 +89,10 @@ def location(self, item): path("pages/", include("zds.pages.urls")), path("galerie/", include("zds.gallery.urls")), path("rechercher/", include("zds.searchv2.urls")), - path("munin/", include(("zds.munin.urls", ""))), + path("munin/", include(("zds.munin.urls", "munin"), namespace="munin")), path("mise-en-avant/", include("zds.featured.urls")), path("notifications/", include("zds.notification.urls")), path("", include(("social_django.urls", "social_django"), namespace="social")), - re_path(r"^munin/", include(("munin.urls", "munin"))), re_path(r"^$", home_view, name="homepage"), re_path(r"^api/", include(("zds.api.urls", "zds.api"), namespace="api")), re_path(r"^oauth2/", include(("oauth2_provider.urls", "oauth2_provider"), namespace="oauth2_provider")),